meerschaum 2.6.17__tar.gz → 2.7.0rc1__tar.gz

{meerschaum-2.6.17/meerschaum.egg-info → meerschaum-2.7.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.17
+Version: 2.7.0rc1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/actions/delete.py

@@ -9,6 +9,7 @@ Functions for deleting elements.
 from __future__ import annotations
 from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List
 
+
 def delete(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -21,7 +22,6 @@ def delete(
 
     """
     from meerschaum.actions import choose_subaction
-    from meerschaum.utils.debug import dprint
     options = {
         'config'     : _delete_config, 
         'pipes'      : _delete_pipes,
@@ -65,19 +65,18 @@ def _complete_delete(
 
 
 def _delete_pipes(
-        debug: bool = False,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    debug: bool = False,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipes and delete their registrations.
-
     """
     from meerschaum import get_pipes
     from meerschaum.utils.prompt import yes_no
-    from meerschaum.utils.formatting import pprint, highlight_pipes
+    from meerschaum.utils.formatting import highlight_pipes
     from meerschaum.utils.warnings import warn
     from meerschaum.actions import actions
     pipes = get_pipes(as_list=True, debug=debug, **kw)
@@ -126,21 +125,22 @@ def _delete_pipes(
     
     return successes > 0, msg
 
+
 def _delete_config(
-        action: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete configuration files.
-
     """
-    import os, shutil
+    import os
+    import shutil
     from meerschaum.utils.prompt import yes_no
-    from meerschaum.config._paths import CONFIG_DIR_PATH, STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
+    from meerschaum.config._paths import STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
     from meerschaum.config._read_config import get_possible_keys, get_keyfile_path
     from meerschaum.utils.debug import dprint
     paths = [p for p in [STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH] if p.exists()]
@@ -157,7 +157,7 @@ def _delete_config(
     else:
         sep = '\n' + '  - '
         answer = yes_no(
-            f"Are you sure you want to delete the following configuration files?" +
+            "Are you sure you want to delete the following configuration files?" +
             f"{sep + sep.join([str(p) for p in paths])}\n",
             default='n', noask=noask, yes=yes
         )
@@ -176,18 +176,18 @@ def _delete_config(
     
     return success, msg
 
+
 def _delete_plugins(
-        action: Optional[List[str]] = None,
-        repository: Optional[str] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    repository: Optional[str] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete plugins from a Meerschaum repository.
-
     """
     from meerschaum.utils.warnings import info
     from meerschaum.plugins import reload_plugins
@@ -206,7 +206,7 @@ def _delete_plugins(
     ) if not force else True
 
     if not answer:
-        return False, f"No plugins deleted."
+        return False, "No plugins deleted."
 
     successes = {}
     for name in action:
@@ -218,27 +218,24 @@ def _delete_plugins(
     reload_plugins(debug=debug)
     return True, "Success"
 
+
 def _delete_users(
-        action: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        shell: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    shell: bool = False,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete users from a Meerschaum instance. Adequate permissions are required.
-
     """
-    from meerschaum import get_connector
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum.utils.prompt import yes_no, prompt
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum.utils.warnings import info
     from meerschaum.core import User
-    from meerschaum.connectors.api import APIConnector
     from meerschaum.utils.formatting import print_tuple
     instance_connector = parse_instance_keys(mrsm_instance)
 
@@ -292,24 +289,25 @@ def _delete_users(
     )
     return True, msg
 
+
 def _delete_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete configured connectors.
     
     Example:
         `delete connectors sql:test`
-
     """
-    import os, pathlib
-    from meerschaum.utils.prompt import yes_no, prompt
+    import os
+    import pathlib
+    from meerschaum.utils.prompt import yes_no
     from meerschaum.connectors.parse import parse_connector_keys
     from meerschaum.config import _config
     from meerschaum.config._edit import write_config
@@ -329,7 +327,7 @@ def _delete_connectors(
     for ck in _keys:
         try:
             conn = parse_connector_keys(ck, debug=debug)
-        except Exception as e:
+        except Exception:
             warn(f"Could not parse connector '{ck}'. Skipping...", stack=False)
             continue
 
@@ -357,26 +355,27 @@ def _delete_connectors(
                     ):
                         try:
                             os.remove(c.database)
-                        except Exception as e:
+                        except Exception:
                             warn(
                                 "Failed to delete database file for connector "
                                 + f"'{c}'. Ignoring...", stack=False
                             )
-        except Exception as e:
+        except Exception:
             pass
         try:
             del cf['meerschaum']['connectors'][c.type][c.label]
-        except Exception as e:
+        except Exception:
             warn(f"Failed to delete connector '{c}' from configuration. Skipping...", stack=False)
 
     write_config(cf, debug=debug)
     return True, "Success"
 
+
 def _complete_delete_connectors(
-        action: Optional[List[str]] = None,
-        line: str = '',
-        **kw: Any
-    ) -> List[str]:
+    action: Optional[List[str]] = None,
+    line: str = '',
+    **kw: Any
+) -> List[str]:
     from meerschaum.config import get_config
     from meerschaum.utils.misc import get_connector_labels
     types = list(get_config('meerschaum', 'connectors').keys())
@@ -401,10 +400,8 @@ def _delete_jobs(
     Remove a job's log files and delete the job's ID.
     
     If the job is running, ask to kill the job first.
-
     """
     from meerschaum.jobs import (
-        Job,
         get_running_jobs,
         get_stopped_jobs,
         get_filtered_jobs,
@@ -460,7 +457,7 @@ def _delete_jobs(
             ### Ensure the running jobs are dead.
             if get_running_jobs(executor_keys, jobs, debug=debug):
                 return False, (
-                    f"Failed to kill running jobs. Please stop these jobs before deleting."
+                    "Failed to kill running jobs. Please stop these jobs before deleting."
                 )
             _to_delete.update(to_stop_jobs)
 
@@ -475,7 +472,7 @@ def _delete_jobs(
         pprint_jobs(_to_delete, nopretty=nopretty)
         if not yes_no(
             "Are you sure you want to delete these jobs?",
-            yes=yes, noask=noask, default='y',
+            yes=yes, noask=noask, default='n',
         ):
             return False, "No jobs were deleted."
 
@@ -569,7 +566,6 @@ def _delete_venvs(
     from meerschaum.utils.venv import venv_exists
     from meerschaum.utils.prompt import yes_no
     from meerschaum.utils.misc import print_options
-    from meerschaum.utils.warnings import warn
 
     venvs_to_skip = ['mrsm']
     venvs = [

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/actions/install.py

@@ -175,7 +175,7 @@ def _install_packages(
             + f" into the virtual environment '{venv}'."
         )
     return False, (
-        f"Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
+        "Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
     )
 
 
@@ -200,7 +200,6 @@ def _install_required(
     from meerschaum.core import Plugin
     from meerschaum.utils.warnings import warn, info
     from meerschaum.connectors.parse import parse_repo_keys
-    from meerschaum.utils.formatting import print_tuple
     from meerschaum.plugins import get_plugins_names
     repo_connector = parse_repo_keys(repository)
 

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/config/_default.py

@@ -137,7 +137,7 @@ default_pipes_config = {
         },
         'verify': {
             'bound_days': 366,
-            'chunk_minutes': 1440,
+            'chunk_minutes': 43200,
         },
     },
     'attributes': {

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/config/_paths.py

@@ -103,7 +103,7 @@ if ENVIRONMENT_VENVS_DIR in os.environ:
     if not _VENVS_DIR_PATH.exists():
         try:
             _VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True)
-        except Exception as e:
+        except Exception:
             print(
                 f"Invalid path set for environment variable '{ENVIRONMENT_VENVS_DIR}':\n"
                 + f"{_VENVS_DIR_PATH}"
@@ -148,6 +148,7 @@ paths = {
     'CACHE_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', '.cache'),
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
     'USERS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'users'),
+    'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.17"
+__version__ = "2.7.0rc1"

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/connectors/api/_pipes.py

@@ -15,7 +15,8 @@ from datetime import datetime
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
-from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
+from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, List, Dict, Tuple
+
 
 def pipe_r_url(
     pipe: mrsm.Pipe
@@ -30,6 +31,7 @@ def pipe_r_url(
         + f"{pipe.connector_keys}/{pipe.metric_key}/{location_key}"
     )
 
+
 def register_pipe(
     self,
     pipe: mrsm.Pipe,
@@ -39,7 +41,6 @@ def register_pipe(
     Returns a tuple of (success_bool, response_dict).
     """
     from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import STATIC_CONFIG
     ### NOTE: if `parameters` is supplied in the Pipe constructor,
     ###       then `pipe.parameters` will exist and not be fetched from the database.
     r_url = pipe_r_url(pipe)
@@ -180,7 +181,7 @@ def sync_pipe(
     from meerschaum.utils.misc import json_serialize_datetime, items_str
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.dataframe import get_numeric_cols, to_json
+    from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
     begin = time.time()
     more_itertools = attempt_import('more_itertools')
     if df is None:

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/connectors/sql/_pipes.py

@@ -460,10 +460,16 @@ def get_create_index_queries(
         else None
     )
     primary_key_constraint_name = (
-        sql_item_name(f'pk_{pipe.target}', self.flavor, None)
+        sql_item_name(f'PK_{pipe.target}', self.flavor, None)
         if primary_key is not None
         else None
     )
+    primary_key_clustered = "CLUSTERED" if _datetime is None else "NONCLUSTERED"
+    datetime_clustered = (
+        "CLUSTERED"
+        if not existing_primary_keys and _datetime is not None
+        else "NONCLUSTERED"
+    )
 
     _id_index_name = (
         sql_item_name(index_names['id'], self.flavor, None)
@@ -474,6 +480,7 @@ def get_create_index_queries(
     _create_space_partition = get_config('system', 'experimental', 'space')
 
     ### create datetime index
+    dt_query = None
     if _datetime is not None:
         if self.flavor == 'timescaledb' and pipe.parameters.get('hypertable', True):
             _id_count = (
@@ -504,19 +511,19 @@ def get_create_index_queries(
                 + 'if_not_exists => true, '
                 + "migrate_data => true);"
             )
-        elif self.flavor == 'mssql':
-            dt_query = (
-                "CREATE "
-                + ("CLUSTERED " if not primary_key else '')
-                + f"INDEX {_datetime_index_name} "
-                + f"ON {_pipe_name} ({_datetime_name})"
-            )
-        else: ### mssql, sqlite, etc.
-            dt_query = (
-                f"CREATE INDEX {_datetime_index_name} "
-                + f"ON {_pipe_name} ({_datetime_name})"
-            )
+        elif _datetime_index_name:
+            if self.flavor == 'mssql':
+                dt_query = (
+                    f"CREATE {datetime_clustered} INDEX {_datetime_index_name} "
+                    f"ON {_pipe_name} ({_datetime_name})"
+                )
+            else:
+                dt_query = (
+                    f"CREATE INDEX {_datetime_index_name} "
+                    + f"ON {_pipe_name} ({_datetime_name})"
+                )
 
+    if dt_query:
         index_queries[_datetime] = [dt_query]
 
     primary_queries = []
@@ -623,7 +630,7 @@ def get_create_index_queries(
                     ),
                     (
                         f"ALTER TABLE {_pipe_name}\n"
-                        f"ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})"
+                        f"ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name})"
                     ),
                 ])
         index_queries[primary_key] = primary_queries
@@ -875,6 +882,7 @@ def get_pipe_data(
     from meerschaum.utils.dtypes import (
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
+        attempt_cast_to_bytes,
         are_dtypes_equal,
     )
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
@@ -891,17 +899,15 @@ def get_pipe_data(
             col: get_pd_type_from_db_type(typ)
             for col, typ in cols_types.items()
         }
-    }
+    } if pipe.enforce else {}
     if dtypes:
         if self.flavor == 'sqlite':
             if not pipe.columns.get('datetime', None):
                 _dt = pipe.guess_datetime()
                 dt = sql_item_name(_dt, self.flavor, None) if _dt else None
-                is_guess = True
             else:
                 _dt = pipe.get_columns('datetime')
                 dt = sql_item_name(_dt, self.flavor, None)
-                is_guess = False
 
             if _dt:
                 dt_type = dtypes.get(_dt, 'object').lower()
@@ -929,7 +935,7 @@ def get_pipe_data(
         col: to_pandas_dtype(typ)
         for col, typ in dtypes.items()
         if col in select_columns and col not in (omit_columns or [])
-    }
+    } if pipe.enforce else {}
     query = self.get_pipe_data_query(
         pipe,
         select_columns=select_columns,
@@ -959,6 +965,11 @@ def get_pipe_data(
         for col, typ in pipe.dtypes.items()
         if typ == 'uuid' and col in dtypes
     ]
+    bytes_columns = [
+        col
+        for col, typ in pipe.dtypes.items()
+        if typ == 'bytes' and col in dtypes
+    ]
 
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
 
@@ -978,6 +989,11 @@ def get_pipe_data(
             continue
         df[col] = df[col].apply(attempt_cast_to_uuid)
 
+    for col in bytes_columns:
+        if col not in df.columns:
+            continue
+        df[col] = df[col].apply(attempt_cast_to_bytes)
+
     if self.flavor == 'sqlite':
         ignore_dt_cols = [
             col
@@ -1339,7 +1355,13 @@ def create_pipe_table_from_df(
     """
     Create a pipe's table from its configured dtypes and an incoming dataframe.
     """
-    from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols, get_uuid_cols
+    from meerschaum.utils.dataframe import (
+        get_json_cols,
+        get_numeric_cols,
+        get_uuid_cols,
+        get_datetime_cols,
+        get_bytes_cols,
+    )
     from meerschaum.utils.sql import get_create_table_queries, sql_item_name
     primary_key = pipe.columns.get('primary', None)
     dt_col = pipe.columns.get('datetime', None)
@@ -1365,6 +1387,18 @@ def create_pipe_table_from_df(
             col: 'numeric'
             for col in get_numeric_cols(df)
         },
+        **{
+            col: 'bytes'
+            for col in get_bytes_cols(df)
+        },
+        **{
+            col: 'datetime64[ns, UTC]'
+            for col in get_datetime_cols(df, timezone_aware=True, timezone_naive=False)
+        },
+        **{
+            col: 'datetime64[ns]'
+            for col in get_datetime_cols(df, timezone_aware=False, timezone_naive=True)
+        },
         **pipe.dtypes
     }
     autoincrement = (
@@ -1455,11 +1489,9 @@ def sync_pipe(
         get_update_queries,
         sql_item_name,
         update_queries,
-        get_create_table_queries,
         get_reset_autoincrement_queries,
     )
     from meerschaum.utils.misc import generate_password
-    from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols, get_uuid_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     from meerschaum import Pipe
@@ -1572,6 +1604,7 @@ def sync_pipe(
         'schema': self.get_pipe_schema(pipe),
     })
 
+    dt_col = pipe.columns.get('datetime', None)
     primary_key = pipe.columns.get('primary', None)
     autoincrement = (
         pipe.parameters.get('autoincrement', False)
@@ -1714,7 +1747,11 @@ def sync_pipe(
             col
             for col_key, col in pipe.columns.items()
             if col and col in existing_cols
-        ]
+        ] if not primary_key else (
+            [dt_col, primary_key]
+            if self.flavor == 'timescaledb' and dt_col and dt_col in update_df.columns
+            else [primary_key]
+        )
         update_queries = get_update_queries(
             pipe.target,
             temp_target,
@@ -1723,7 +1760,8 @@ def sync_pipe(
             upsert=upsert,
             schema=self.get_pipe_schema(pipe),
             patch_schema=self.internal_schema,
-            datetime_col=pipe.columns.get('datetime', None),
+            datetime_col=(dt_col if dt_col in update_df.columns else None),
+            identity_insert=(autoincrement and primary_key in update_df.columns),
             debug=debug,
         )
         update_success = all(
@@ -1841,7 +1879,6 @@ def sync_pipe_inplace(
         session_execute,
         update_queries,
     )
-    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
         get_pd_type_from_db_type,
     )
@@ -2061,6 +2098,7 @@ def sync_pipe_inplace(
     ) if not (upsert or static) else new_cols_types
 
     common_cols = [col for col in new_cols if col in backtrack_cols_types]
+    primary_key = pipe.columns.get('primary', None)
     on_cols = {
         col: new_cols.get(col)
         for col_key, col in pipe.columns.items()
@@ -2071,7 +2109,7 @@ def sync_pipe_inplace(
             and col in backtrack_cols_types
             and col in new_cols
         )
-    }
+    } if not primary_key else {primary_key: new_cols.get(primary_key)}
 
     null_replace_new_cols_str = (
         ', '.join([

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/connectors/sql/_sql.py

@@ -790,7 +790,12 @@ def to_sql(
         truncate_item_name,
         DROP_IF_EXISTS_FLAVORS,
     )
-    from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols, get_uuid_cols
+    from meerschaum.utils.dataframe import (
+        get_json_cols,
+        get_numeric_cols,
+        get_uuid_cols,
+        get_bytes_cols,
+    )
     from meerschaum.utils.dtypes import are_dtypes_equal, quantize_decimal, coerce_timezone
     from meerschaum.utils.dtypes.sql import (
         NUMERIC_PRECISION_FLAVORS,

{meerschaum-2.6.17 → meerschaum-2.7.0rc1}/meerschaum/connectors/valkey/_pipes.py

@@ -46,9 +46,20 @@ def serialize_document(doc: Dict[str, Any]) -> str:
     -------
     A serialized string for the document.
     """
+    from meerschaum.utils.dtypes import serialize_bytes
     return json.dumps(
         doc,
-        default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+        default=(
+            lambda x: (
+                json_serialize_datetime(x)
+                if hasattr(x, 'tzinfo')
+                else (
+                    serialize_bytes(x)
+                    if isinstance(x, bytes)
+                    else str(x)
+                )
+            )
+        ),
         separators=(',', ':'),
         sort_keys=True,
     )