meerschaum 1.5.7__tar.gz → 1.5.18__tar.gz

{meerschaum-1.5.7/meerschaum.egg-info → meerschaum-1.5.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 1.5.7
+Version: 1.5.18
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -20,7 +20,6 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS
 Classifier: Programming Language :: SQL
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -33,7 +32,6 @@ Provides-Extra: required
 Provides-Extra: minimal
 Provides-Extra: formatting
 Provides-Extra: _required
-Provides-Extra: iot
 Provides-Extra: drivers
 Provides-Extra: _drivers
 Provides-Extra: cli

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/__init__.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Copyright 2021 Bennett Meares
+Copyright 2023 Bennett Meares
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -22,9 +22,20 @@ from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils import get_pipes
 from meerschaum.utils.venv import Venv
+from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
 from meerschaum.connectors import get_connector
-from meerschaum.config import __version__
+from meerschaum.config import __version__, get_config
+from meerschaum.utils.packages import attempt_import
 
 __pdoc__ = {'gui': False, 'api': False, 'core': False,}
-__all__ = ("Pipe", "get_pipes", "get_connector", "Plugin",)
+__all__ = (
+    "Pipe",
+    "get_pipes",
+    "get_connector",
+    "Plugin",
+    "Venv",
+    "Plugin",
+    "pprint",
+    "attempt_import",
+)

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/_internal/arguments/_parse_arguments.py

@@ -96,11 +96,12 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
             args_dict['text'] = shlex.join(sysargs)
         except Exception as _e:
             args_dict['text'] = ' '.join(sysargs)
+        args_dict[STATIC_CONFIG['system']['arguments']['failure_key']] = e
         unknown = []
 
     false_flags = [arg for arg, val in args_dict.items() if val is False]
     for arg in false_flags:
-        args_dict.pop(arg, None)
+        _ = args_dict.pop(arg, None)
 
     args_dict['sysargs'] = sysargs
     args_dict['filtered_sysargs'] = filtered_sysargs

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/_internal/arguments/_parser.py

@@ -11,12 +11,17 @@ import sys
 import argparse, json
 from meerschaum.utils.typing import Union, Dict, List, Any, Tuple, Callable
 from meerschaum.utils.misc import string_to_dict
+from meerschaum.utils.warnings import error
 
+class InvalidArgumentException(Exception):
+    """
+    Raised when argparse failes to parse arguments.
+    """
 
 _original_argparse_error = argparse.ArgumentParser.error
 _original_argparse_parse_known_args = argparse.ArgumentParser.parse_known_args
-def _new_argparse_error(self, message):
-    raise argparse.ArgumentError(message)
+def _new_argparse_error(message: str):
+    error(message, InvalidArgumentException, stack=False)
 
 class ArgumentParser(argparse.ArgumentParser):
     """Override the built-in `argparse` error handling."""
@@ -229,11 +234,6 @@ groups['sync'].add_argument(
         "Specify the chunksize for syncing and retrieving data. Defaults to 900."
     ),
 )
-groups['sync'].add_argument(
-    '--sync-chunks', action='store_true',
-    help="Sync chunks while fetching data instead of waiting until all have arrived. " +
-    "Similar to --async. WARNING! This can be very dangerous when used with --async.",
-)
 groups['sync'].add_argument(
     '--skip-check-existing', '--allow-duplicates', action='store_true',
     help = (

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/_internal/entry.py

@@ -23,12 +23,29 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
     """
     from meerschaum._internal.arguments import parse_arguments
+    from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
         sysargs = []
     if not isinstance(sysargs, list):
         import shlex
         sysargs = shlex.split(sysargs)
     args = parse_arguments(sysargs)
+    argparse_exception = args.get(
+        STATIC_CONFIG['system']['arguments']['failure_key'],
+        None,
+    )
+    if argparse_exception is not None:
+        args_text = args.get('text', '')
+        if not args_text.startswith('show arguments'):
+            return (
+                False,
+                (
+                    "Invalid arguments:"
+                    + (f"\n{args_text}" if args_text else '')
+                    + f"\n    {argparse_exception}"
+                )
+            )
+
     if args.get('schedule', None):
         from meerschaum.utils.schedule import schedule_function
         return schedule_function(entry_with_args, args['schedule'], **args)

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/actions/bootstrap.py

@@ -234,7 +234,7 @@ def _bootstrap_connectors(
 
     """
     from meerschaum.connectors.parse import is_valid_connector_keys
-    from meerschaum.connectors import connectors, get_connector
+    from meerschaum.connectors import connectors, get_connector, types, custom_types
     from meerschaum.utils.prompt import prompt, yes_no, choose
     from meerschaum.config import get_config
     from meerschaum.config._edit import write_config
@@ -258,11 +258,12 @@ def _bootstrap_connectors(
     try:
         _type = choose(
             (
-                'Please choose a connector type.\n' +
-                'For more information on connectors, please visit https://meerschaum.io/reference/connectors'
+                'Please choose a connector type.\n'
+                + 'For more information on connectors, '
+                + 'please visit https://meerschaum.io/reference/connectors'
             ),
             sorted(list(connectors)),
-            default='sql'
+            default = 'sql'
         )
     except KeyboardInterrupt:
         return abort_tuple
@@ -270,10 +271,12 @@ def _bootstrap_connectors(
     if _clear:
         clear_screen(debug=debug)
 
-    _label_choices = sorted(
-        [label for label in get_config('meerschaum', 'connectors', _type)
-            if label != 'default']
-    )
+    existing_labels = get_config('meerschaum', 'connectors', _type, warn=False) or []
+    _label_choices = sorted([
+        label
+        for label in existing_labels
+        if label != 'default' and label is not None
+    ])
     new_connector_label = 'New connector'
     _label_choices.append(new_connector_label)
     while True:
@@ -281,9 +284,14 @@ def _bootstrap_connectors(
             _label = prompt(f"New label for '{_type}' connector:")
         except KeyboardInterrupt:
             return abort_tuple
-        if _label in get_config('meerschaum', 'connectors', _type):
+        if _label in existing_labels:
             warn(f"Connector '{_type}:{_label}' already exists.", stack=False)
-            overwrite = yes_no(f"Do you want to overwrite connector '{_type}:{_label}'?", default='n', yes=yes, noask=noask)
+            overwrite = yes_no(
+                f"Do you want to overwrite connector '{_type}:{_label}'?",
+                default = 'n',
+                yes = yes,
+                noask = noask,
+            )
             if not overwrite and not force:
                 return False, f"No changes made to connector configuration."
                 break
@@ -292,14 +300,19 @@ def _bootstrap_connectors(
         else:
             break
 
+    cls = types.get(_type)
+    cls_required_attrs = getattr(cls, 'REQUIRED_ATTRIBUTES', [])
+    type_attributes = connector_attributes.get(_type, {'required': cls_required_attrs})
+
     new_attributes = {}
-    if 'flavors' in connector_attributes[_type]:
+    if 'flavors' in type_attributes:
         try:
             flavor = choose(
                 f"Flavor for connector '{_type}:{_label}':",
-                sorted(list(connector_attributes[_type]['flavors'])),
+                sorted(list(type_attributes['flavors'])),
                 default = (
-                    'timescaledb' if 'timescaledb' in connector_attributes[_type]['flavors']
+                    'timescaledb'
+                    if 'timescaledb' in type_attributes['flavors']
                     else None
                 )
             )
@@ -307,10 +320,10 @@ def _bootstrap_connectors(
             return abort_tuple
         new_attributes['flavor'] = flavor
         required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
-        default = connector_attributes[_type]['flavors'][flavor]['defaults']
+        default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
-        required = sorted(list(connector_attributes[_type]['required']))
-        default = connector_attributes[_type]['default']
+        required = sorted(list(type_attributes.get('required', {})))
+        default = type_attributes.get('default', {})
     info(
         f"Please answer the following questions to configure the new connector '{_type}:{_label}'."
         + '\n' + "Press [Ctrl + C] to skip."

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/actions/edit.py

@@ -203,6 +203,7 @@ def _edit_users(
     from meerschaum.utils.prompt import prompt, yes_no, get_password, get_email
     from meerschaum.utils.misc import edit_file
     from meerschaum.config._paths import USERS_CACHE_RESOURCES_PATH
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.yaml import yaml
     import os, pathlib
     instance_connector = parse_instance_keys(mrsm_instance)
@@ -214,7 +215,10 @@ def _edit_users(
         ### Change the password
         password = ''
         if yes_no(f"Change the password for user '{username}'?", default='n', yes=yes, noask=noask):
-            password = get_password(username, minimum_length=7)
+            password = get_password(
+                username,
+                minimum_length = STATIC_CONFIG['users']['min_password_length'],
+            )
 
         ## Make an admin
         _type = ''

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/actions/register.py

@@ -256,7 +256,7 @@ def _register_users(
     Register a new user to a Meerschaum instance.
     """
     from meerschaum.config import get_config
-    from meerschaum.config.static import _static_config
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum import get_connector
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.debug import dprint
@@ -274,7 +274,7 @@ def _register_users(
     ### filter out existing users
     nonregistered_users = []
     for username in action:
-        min_len = _static_config()['users']['min_username_length']
+        min_len = STATIC_CONFIG['users']['min_username_length']
         if len(username) < min_len:
             warn(
                 f"Username '{username}' is too short (less than {min_len} characters). Skipping...",
@@ -296,7 +296,7 @@ def _register_users(
             username = _user.username
             password = get_password(
                 username,
-                minimum_length = _static_config()['users']['min_password_length']
+                minimum_length = STATIC_CONFIG['users']['min_password_length']
             )
             email = get_email(username, allow_omit=True)
         except Exception as e:

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/api/dash/callbacks/register.py

@@ -12,7 +12,7 @@ from meerschaum.api.dash import dash_app, debug, active_sessions
 from dash.dependencies import Input, Output, State, ALL, MATCH
 from dash.exceptions import PreventUpdate
 from meerschaum.core import User
-from meerschaum.config.static import _static_config
+from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.packages import attempt_import
 dash = attempt_import('dash', check_update=CHECK_UPDATE)
 from fastapi.exceptions import HTTPException
@@ -24,7 +24,7 @@ from fastapi.exceptions import HTTPException
 def validate_username(username):
     if not username:
         raise PreventUpdate
-    valid = (len(username) >= _static_config()['users']['min_username_length'])
+    valid = (len(username) >= STATIC_CONFIG['users']['min_username_length'])
     if not valid:
         return valid, not valid
     conn = get_api_connector()
@@ -40,7 +40,7 @@ def validate_username(username):
 def validate_password(password):
     if not password:
         raise PreventUpdate
-    valid = (len(password) >= _static_config()['users']['min_password_length'])
+    valid = (len(password) >= STATIC_CONFIG['users']['min_password_length'])
     return valid, not valid
 
 @dash_app.callback(
@@ -75,6 +75,11 @@ def register_button_click(
     form_class = 'form-control'
     from meerschaum.api.routes._login import login
     conn = get_api_connector()
+    if not username or not password:
+        success, msg = False, "Invalid username or password."
+        form_class += ' is-invalid'
+        return {}, form_class, dash.no_update
+
     user = User(username, password, email=email, instance=conn)
     user_id = conn.get_user_id(user, debug=debug)
     if user_id is not None:

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/config/_default.py

@@ -51,13 +51,6 @@ default_meerschaum_config = {
                 'protocol' : 'https',
             },
         },
-        'mqtt' : {
-            'default'      : connector_attributes['mqtt']['default'],
-            'main'         : {
-                'host'     : 'localhost',
-                'port'     : 1883,
-            },
-        },
     },
 }
 default_system_config = {
@@ -66,7 +59,7 @@ default_system_config = {
             'pandas'           : 'pandas',
         },
         'sql' : {
-            'chunksize'        : 1000000,
+            'chunksize'        : 100000,
             'poolclass'        : 'sqlalchemy.pool.QueuePool',
             'create_engine'    : {
                 'method'       : 'multi',

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.5.7"
+__version__ = "1.5.18"

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/config/static/__init__.py

@@ -57,6 +57,7 @@ STATIC_CONFIG = {
                 ']'
             ),
             'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
+            'failure_key': '_argparse_exception',
         },
         'urls': {
             'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',
@@ -88,9 +89,9 @@ STATIC_CONFIG = {
             'default': 'pbkdf2_sha256',
             'pbkdf2_sha256__default_rounds': 30000,
         },
-        'min_username_length': 3,
+        'min_username_length': 1,
         'max_username_length': 26,
-        'min_password_length': 5,
+        'min_password_length': 1,
     },
     'plugins': {
         'repo_separator': '@',
@@ -101,6 +102,7 @@ STATIC_CONFIG = {
         'dtypes': {
             'min_ratio_columns_changed_for_full_astype': 0.5,
         },
+        'exists_timeout_seconds': 5.0,
     },
     'setup': {
         'name': 'meerschaum',

{meerschaum-1.5.7 → meerschaum-1.5.18}/meerschaum/connectors/Connector.py

@@ -3,9 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Generic Connector parent class. Reads attributes from the configuration.
-Consult implemented child classes like SQLConnector, APIConnector, MQTTConnector, or
-PluginConnector for further details.
+Define the parent `Connector` class.
 """
 
 from __future__ import annotations
@@ -13,6 +11,11 @@ import abc
 import copy
 from meerschaum.utils.typing import Iterable, Optional, Any, Union, List, Dict
 
+class InvalidAttributesError(Exception):
+    """
+    Raised when the incorrect attributes are set in the Connector.
+    """
+
 class Connector(metaclass=abc.ABCMeta):
     """
     The base connector class to hold connection attributes,
@@ -27,16 +30,10 @@ class Connector(metaclass=abc.ABCMeta):
         Parameters
         ----------
         type: str
-            The type of the connection. Used as a key in config.yaml to get attributes.
-            Supported values are 'sql', 'api', 'mqtt', 'plugin'.
+            The `type` of the connector (e.g. `sql`, `api`, `plugin`).
 
         label: str
-            The label for the connection. Used as a key within config.yaml
-
-        pandas: str
-            Custom pandas implementation name.
-            E.g. May change to modin.pandas.
-            **NOTE:** This is experimental!
+            The `label` for the connector.
 
         Run `mrsm edit config` and to edit connectors in the YAML file:
 
@@ -51,22 +48,40 @@ class Connector(metaclass=abc.ABCMeta):
         """
         self._original_dict = copy.deepcopy(self.__dict__)
         self._set_attributes(type=type, label=label, **kw)
+        self.verify_attributes(getattr(self, 'REQUIRED_ATTRIBUTES', None))
 
     def _reset_attributes(self):
         self.__dict__ = self._original_dict
 
     def _set_attributes(
             self,
-            type: Optional[str] = None,
-            label: str = "main",
-            pandas: Optional[str] = None,
+            *args,
             inherit_default: bool = True,
             **kw: Any
         ):
+        from meerschaum.config.static import STATIC_CONFIG
         from meerschaum.utils.warnings import error
+
+        self._attributes = {}
+
+        default_label = STATIC_CONFIG['connectors']['default_label']
+
+        ### NOTE: Support the legacy method of explicitly passing the type.
+        label = kw.get('label', None)
+        if label is None:
+            if len(args) == 2:
+                label = args[1]
+            elif len(args) == 0:
+                label = None
+            else:
+                label = args[0]
+
         if label == 'default':
-            error("Label cannot be 'default'. Did you mean 'main'?")
-        self.type, self.label = type, label
+            error(
+                f"Label cannot be 'default'. Did you mean '{default_label}'?",
+                InvalidAttributesError,
+            )
+        self.__dict__['label'] = label
 
         from meerschaum.config import get_config
         conn_configs = copy.deepcopy(get_config('meerschaum', 'connectors'))
@@ -77,19 +92,22 @@ class Connector(metaclass=abc.ABCMeta):
             inherit_from = 'default'
             if self.type in conn_configs and inherit_from in conn_configs[self.type]:
                 _inherit_dict = copy.deepcopy(conn_configs[self.type][inherit_from])
-                self.__dict__.update(_inherit_dict)
+                self._attributes.update(_inherit_dict)
 
-        ### load user config into self.__dict__
+        ### load user config into self._attributes
         if self.type in conn_configs and self.label in conn_configs[self.type]:
-            self.__dict__.update(conn_configs[self.type][self.label])
+            self._attributes.update(conn_configs[self.type][self.label])
 
-        ### load system config into self.sys_config
+        ### load system config into self._sys_config
         ### (deep copy so future Connectors don't inherit changes)
         if self.type in connector_config:
-            self.sys_config = copy.deepcopy(connector_config[self.type])
+            self._sys_config = copy.deepcopy(connector_config[self.type])
 
         ### add additional arguments or override configuration
-        self.__dict__.update(kw)
+        self._attributes.update(kw)
+
+        ### finally, update __dict__ with _attributes.
+        self.__dict__.update(self._attributes)
 
 
     def verify_attributes(
@@ -131,19 +149,66 @@ class Connector(metaclass=abc.ABCMeta):
         if len(missing_attributes) > 0:
             error(
                 (
-                    "Please provide connection configuration for connector "
-                    + f"'{self.type}:{self.label}' "
-                    + "in the configuration file (open with `mrsm edit config`) or as arguments "
-                    + "for the Connector.\n\n"
-                    + f"Missing {items_str(list(missing_attributes))}."
+                    f"Missing {items_str(list(missing_attributes))} "
+                    + f"for connector '{self.type}:{self.label}'."
                 ),
+                InvalidAttributesError,
                 silent = True,
                 stack = False
             )
 
 
     def __str__(self):
+        """
+        When cast to a string, return type:label.
+        """
         return f"{self.type}:{self.label}"
 
     def __repr__(self):
+        """
+        Represent the connector as type:label.
+        """
         return str(self)
+
+    @property
+    def meta(self) -> Dict[str, Any]:
+        """
+        Return the keys needed to reconstruct this Connector.
+        """
+        _meta = {
+            key: value
+            for key, value in self.__dict__.items()
+            if not str(key).startswith('_')
+        }
+        _meta.update({
+            'type': self.type,
+            'label': self.label,
+        })
+        return _meta
+
+
+    @property
+    def type(self) -> str:
+        """
+        Return the type for this connector.
+        """
+        _type = self.__dict__.get('type', None)
+        if _type is None:
+            import re
+            _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
+            self.__dict__['type'] = _type
+        return _type
+
+
+    @property
+    def label(self) -> str:
+        """
+        Return the label for this connector.
+        """
+        _label = self.__dict__.get('label', None)
+        if _label is None:
+            from meerschaum.config.static import STATIC_CONFIG
+            _label = STATIC_CONFIG['connectors']['default_label']
+            self.__dict__['label'] = _label
+        return _label
+