meerschaum 2.8.1__tar.gz → 2.8.3__tar.gz

{meerschaum-2.8.1/meerschaum.egg-info → meerschaum-2.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.8.1
+Version: 2.8.3
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/api/__init__.py

@@ -21,6 +21,7 @@ from meerschaum.plugins import _api_plugins
 from meerschaum.utils.warnings import warn, dprint
 from meerschaum.utils.threading import RLock
 from meerschaum.utils.misc import is_pipe_registered
+from meerschaum.connectors.parse import parse_instance_keys
 
 from meerschaum import __version__ as version
 __version__ = version
@@ -99,7 +100,6 @@ production = get_uvicorn_config().get('production', False)
 _include_dash = (not no_dash)
 docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
 
-connector = None
 default_instance_keys = None
 _instance_connectors = defaultdict(lambda: None)
 def get_api_connector(instance_keys: Optional[str] = None):
@@ -130,12 +130,19 @@ def get_api_connector(instance_keys: Optional[str] = None):
         )
 
     with _locks[f'instance-{instance_keys}']:
-        connector = _instance_connectors[instance_keys]
-        if connector is None:
-            from meerschaum.connectors.parse import parse_instance_keys
-            connector = parse_instance_keys(instance_keys, debug=debug)
-            _instance_connectors[instance_keys] = connector
-    return connector
+        if _instance_connectors[instance_keys] is None:
+            try:
+                is_valid_connector = True
+                _instance_connectors[instance_keys] = parse_instance_keys(instance_keys, debug=debug)
+            except Exception:
+                is_valid_connector = False
+
+            if not is_valid_connector:
+                raise fastapi.HTTPException(
+                    status_code=422,
+                    detail="Invalid instance keys.",
+                )
+    return _instance_connectors[instance_keys]
 
 
 cache_connector = None
@@ -164,7 +171,6 @@ def get_cache_connector(connector_keys: Optional[str] = None):
         return None
 
     if cache_connector is None:
-        from meerschaum.connectors.parse import parse_instance_keys
         cache_connector = parse_instance_keys(connector_keys)
 
     if debug:

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/api/routes/_pipes.py

@@ -311,7 +311,7 @@ def get_sync_time(
     newest: bool = True,
     remote: bool = False,
     round_down: bool = True,
-    debug: bool = False,
+    instance_keys: Optional[str] = None,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
@@ -322,12 +322,11 @@ def get_sync_time(
     """
     if location_key == '[None]':
         location_key = None
-    pipe = get_pipe(connector_keys, metric_key, location_key)
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
     sync_time = pipe.get_sync_time(
-        params = params,
-        newest = newest,
-        debug = debug,
-        round_down = round_down,
+        params=params,
+        newest=newest,
+        round_down=round_down,
     )
     if isinstance(sync_time, datetime):
         sync_time = sync_time.isoformat()
@@ -339,7 +338,8 @@ def sync_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
-    data: dict = None,
+    data: Union[List[Dict[Any, Any]], Dict[Any, Any]],
+    instance_keys: Optional[str] = None,
     check_existing: bool = True,
     blocking: bool = True,
     force: bool = False,
@@ -354,9 +354,9 @@ def sync_pipe(
     Add data to an existing Pipe.
     See [`meerschaum.Pipe.sync`](https://docs.meerschaum.io/meerschaum.html#Pipe.sync).
     """
-    if data is None:
-        data = {}
-    pipe = get_pipe(connector_keys, metric_key, location_key)
+    if not data:
+        return [True, "No data to sync."]
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
     if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
         raise fastapi.HTTPException(
             status_code=409,
@@ -365,21 +365,16 @@ def sync_pipe(
 
     if not pipe.columns and columns is not None:
         pipe.columns = json.loads(columns)
-    if not pipe.columns and not is_pipe_registered(pipe, pipes(refresh=True)):
-        raise fastapi.HTTPException(
-            status_code=409,
-            detail="Pipe must be registered with index columns specified."
-        )
 
-    result = list(pipe.sync(
+    success, msg = pipe.sync(
         data,
         debug=debug,
         check_existing=check_existing,
         blocking=blocking,
         force=force,
         workers=workers,
-    ))
-    return result
+    )
+    return list((success, msg))
 
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
@@ -543,7 +538,7 @@ def get_pipe_csv(
     if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
         raise fastapi.HTTPException(
             status_code=409,
-            detail="Pipe must be registered with the datetime column specified."
+            detail="Pipe must be registered."
         )
 
     dt_col = pipe.columns.get('datetime', None)

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/config/_default.py

@@ -173,6 +173,9 @@ default_pipes_config = {
     'sync': {
         'filter_params_index_limit': 250,
     },
+    'verify': {
+        'max_chunks_syncs': 3,
+    },
 }
 default_plugins_config = {}
 default_experimental_config = {

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.8.1"
+__version__ = "2.8.3"

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/config/static/__init__.py

@@ -137,7 +137,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'pbkdf2_sha256__default_rounds': 1_000_000,
         },
         'min_username_length': 1,
-        'max_username_length': 26,
+        'max_username_length': 60,
         'min_password_length': 5,
     },
     'plugins': {

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/connectors/sql/_cli.py

@@ -100,7 +100,13 @@ def _cli_exit(
     ### yet defined (e.g. 'sql:local').
     cli_arg_str = self.DATABASE_URL
     if self.flavor in ('sqlite', 'duckdb'):
-        cli_arg_str = str(self.database)
+        cli_arg_str = (
+            str(self.database)
+            if 'database' in self.__dict__
+            else self.parse_uri(self.URI).get('database', None)
+        )
+        if not cli_arg_str:
+            raise ValueError(f"Cannot determine database from connector '{self}'.")
     if cli_arg_str.startswith('postgresql+psycopg://'):
         cli_arg_str = cli_arg_str.replace('postgresql+psycopg://', 'postgresql://')
 

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/connectors/sql/_create_engine.py

@@ -256,8 +256,11 @@ def create_engine(
         ) if not _uri else _uri
 
         ### Sometimes the timescaledb:// flavor can slip in.
-        if _uri and self.flavor in ('timescaledb',) and self.flavor in _uri:
-            engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
+        if _uri and self.flavor in _uri:
+            if self.flavor == 'timescaledb':
+                engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
+            elif _uri.startswith('postgresql://'):
+                engine_str = engine_str.replace('postgresql://', 'postgresql+psycopg2://')
 
     if debug:
         dprint(
@@ -303,7 +306,7 @@ def create_engine(
             echo         = debug,
             **_create_engine_args
         )
-    except Exception as e:
+    except Exception:
         warn(f"Failed to create connector '{self}':\n{traceback.format_exc()}", stack=False)
         engine = None
 

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/connectors/sql/_users.py

@@ -43,7 +43,9 @@ def register_user(
         'password_hash': user.password_hash,
         'user_type': user.type,
         'attributes': (
-            json.dumps(user.attributes) if self.flavor not in json_flavors else user.attributes
+            json.dumps(user.attributes)
+            if self.flavor not in json_flavors
+            else user.attributes
         ),
     }
     if old_id is not None:
@@ -109,7 +111,7 @@ def edit_user(
     user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
     if user_id is None:
         return False, (
-            f"User '{user.username}' does not exist. " +
+            f"User '{user.username}' does not exist. "
             f"Register user '{user.username}' before editing."
         )
     user.user_id = user_id

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/core/Pipe/_verify.py

@@ -7,6 +7,7 @@ Verify the contents of a pipe by resyncing its interval.
 """
 
 from datetime import datetime, timedelta
+import time
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, Dict
@@ -183,6 +184,7 @@ def verify(
         )
     )
     message_header = f"{begin_to_print} - {end_to_print}"
+    max_chunks_syncs = mrsm.get_config('pipes', 'verify', 'max_chunks_syncs')
 
     info(
         f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk"
@@ -219,9 +221,9 @@ def verify(
                 debug=debug,
             )
             checked_rows_str = (
-                f"checked {existing_rowcount} row"
+                f"checked {existing_rowcount:,} row"
                 + ("s" if existing_rowcount != 1 else '')
-                + f" vs {remote_rowcount} remote"
+                + f" vs {remote_rowcount:,} remote"
             )
             if (
                 existing_rowcount is not None
@@ -239,19 +241,26 @@ def verify(
                     f"Row-counts are out-of-sync ({checked_rows_str})."
                 )
 
-        chunk_success, chunk_msg = self.sync(
-            begin=chunk_begin,
-            end=chunk_end,
-            params=params,
-            workers=_workers,
-            debug=debug,
-            **kwargs
-        ) if do_sync else (chunk_success, chunk_msg)
+        num_syncs = 0
+        while num_syncs < max_chunks_syncs:
+            chunk_success, chunk_msg = self.sync(
+                begin=chunk_begin,
+                end=chunk_end,
+                params=params,
+                workers=_workers,
+                debug=debug,
+                **kwargs
+            ) if do_sync else (chunk_success, chunk_msg)
+            if chunk_success:
+                break
+            num_syncs += 1
+            time.sleep(num_syncs**2)
         chunk_msg = chunk_msg.strip()
         if ' - ' not in chunk_msg:
             chunk_label = f"{chunk_begin} - {chunk_end}"
             chunk_msg = f'Verified chunk for {self}:\n{chunk_label}\n{chunk_msg}'
         mrsm.pprint((chunk_success, chunk_msg))
+
         return chunk_begin_and_end, (chunk_success, chunk_msg)
 
     ### If we have more than one chunk, attempt to sync the first one and return if its fails.

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/jobs/_Job.py

@@ -30,7 +30,7 @@ if TYPE_CHECKING:
     from meerschaum.jobs._Executor import Executor
 
 BANNED_CHARS: List[str] = [
-    ',', ';', "'", '"', '.', '$', '#', '=', '*', '&', '!', '`', '~',
+    ',', ';', "'", '"', '$', '#', '=', '*', '&', '!', '`', '~',
 ]
 RESTART_FLAGS: List[str] = [
     '-s',

{meerschaum-2.8.1 → meerschaum-2.8.3}/meerschaum/utils/dataframe.py

@@ -222,7 +222,7 @@ def filter_unseen_df(
     numeric_cols_precisions_scales = {
         col: get_numeric_precision_scale(None, typ)
         for col, typ in dtypes.items()
-        if col and typ and typ.startswith('numeric')
+        if col and str(typ).lower().startswith('numeric')
     }
 
     dt_dtypes = {

{meerschaum-2.8.1 → meerschaum-2.8.3/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.8.1
+Version: 2.8.3
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares