meerschaum 2.8.3__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl

meerschaum/connectors/api/_plugins.py

@@ -7,21 +7,25 @@ Manage plugins via the API connector
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, Mapping, Sequence
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, List, Dict
+
 
 def plugin_r_url(
-        plugin: Union[meerschaum.core.Plugin, str]
-    ) -> str:
+    plugin: Union[mrsm.core.Plugin, str],
+) -> str:
     """Generate a relative URL path from a Plugin."""
     from meerschaum.config.static import STATIC_CONFIG
     return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
 
+
 def register_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        make_archive: bool = True,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    make_archive: bool = True,
+    debug: bool = False,
+) -> SuccessTuple:
     """Register a plugin and upload its archive."""
     import json
     archive_path = plugin.make_tar(debug=debug) if make_archive else plugin.archive_path
@@ -34,27 +38,30 @@ def register_plugin(
     r_url = plugin_r_url(plugin)
     try:
         response = self.post(r_url, files=files, params=metadata, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to register plugin '{plugin}'."
     finally:
         file_pointer.close()
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
 
+
 def install_plugin(
-        self,
-        name: str,
-        skip_deps: bool = False,
-        force: bool = False,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    name: str,
+    skip_deps: bool = False,
+    force: bool = False,
+    debug: bool = False
+) -> SuccessTuple:
     """Download and attempt to install a plugin from the API."""
-    import os, pathlib, json
+    import os
+    import pathlib
+    import json
     from meerschaum.core import Plugin
     from meerschaum.config._paths import PLUGINS_TEMP_RESOURCES_PATH
     from meerschaum.utils.debug import dprint
@@ -75,41 +82,39 @@ def install_plugin(
                 success, msg = tuple(j)
             elif isinstance(j, dict) and 'detail' in j:
                 success, msg = False, fail_msg
-        except Exception as e:
+        except Exception:
             success, msg = False, fail_msg
         return success, msg
     plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
     return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
 
+
 def get_plugins(
-        self,
-        user_id : Optional[int] = None,
-        search_term : Optional[str] = None,
-        debug : bool = False
-    ) -> Sequence[str]:
+    self,
+    user_id: Optional[int] = None,
+    search_term: Optional[str] = None,
+    debug: bool = False
+) -> List[str]:
     """Return a list of registered plugin names.
 
     Parameters
     ----------
-    user_id :
+    user_id: Optional[int], default None
         If specified, return all plugins from a certain user.
-    user_id : Optional[int] :
-         (Default value = None)
-    search_term : Optional[str] :
-         (Default value = None)
-    debug : bool :
-         (Default value = False)
+
+    search_term: Optional[str], default None
+        If specified, return plugins beginning with this string.
 
     Returns
     -------
-
+    A list of plugin names.
     """
     import json
-    from meerschaum.utils.warnings import warn, error
+    from meerschaum.utils.warnings import error
     from meerschaum.config.static import STATIC_CONFIG
     response = self.get(
         STATIC_CONFIG['api']['endpoints']['plugins'],
-        params = {'user_id' : user_id, 'search_term' : search_term},
+        params = {'user_id': user_id, 'search_term': search_term},
         use_token = True,
         debug = debug
     )
@@ -120,11 +125,12 @@ def get_plugins(
         error(response.text)
     return plugins
 
+
 def get_plugin_attributes(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> Mapping[str, Any]:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> Dict[str, Any]:
     """
     Return a plugin's attributes.
     """
@@ -136,7 +142,7 @@ def get_plugin_attributes(
     if isinstance(attributes, str) and attributes and attributes[0] == '{':
         try:
             attributes = json.loads(attributes)
-        except Exception as e:
+        except Exception:
             pass
     if not isinstance(attributes, dict):
         error(response.text)
@@ -145,23 +151,23 @@ def get_plugin_attributes(
         return {}
     return attributes
 
+
 def delete_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> SuccessTuple:
     """Delete a plugin from an API repository."""
     import json
     r_url = plugin_r_url(plugin)
     try:
         response = self.delete(r_url, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to delete plugin '{plugin}'."
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
-

meerschaum/connectors/api/_request.py

@@ -95,7 +95,7 @@ def make_request(
     return self.session.request(
         method.upper(),
         request_url,
-        headers = headers,
+        headers=headers,
         **kwargs
     )
 

meerschaum/connectors/parse.py

@@ -140,7 +140,6 @@ def is_valid_connector_keys(
     """
     try:
         success = parse_connector_keys(keys, construct=False) is not None
-    except Exception as e:
+    except Exception:
         success = False
     return success
-

meerschaum/connectors/sql/_SQLConnector.py

@@ -184,7 +184,7 @@ class SQLConnector(Connector):
         ### ensure flavor and label are set accordingly
         if 'flavor' not in self.__dict__:
             if flavor is None and 'uri' not in self.__dict__:
-                raise Exception(
+                raise ValueError(
                     f"    Missing flavor. Provide flavor as a key for '{self}'."
                 )
             self.flavor = flavor or self.parse_uri(self.__dict__['uri']).get('flavor', None)

meerschaum/core/Pipe/_data.py

@@ -88,9 +88,8 @@ def get_data(
     limit: Optional[int], default None
         If provided, cap the dataframe to this many rows.
 
-    fresh: bool, default True
+    fresh: bool, default False
         If `True`, skip local cache and directly query the instance connector.
-        Defaults to `True`.
 
     debug: bool, default False
         Verbosity toggle.

meerschaum/core/Pipe/_verify.py

@@ -119,7 +119,7 @@ def verify(
     begin, end = self.parse_date_bounds(begin, end)
     cannot_determine_bounds = bounded and begin is None and end is None
 
-    if cannot_determine_bounds:
+    if cannot_determine_bounds and not check_rowcounts_only:
         warn(f"Cannot determine sync bounds for {self}. Syncing instead...", stack=False)
         sync_success, sync_msg = self.sync(
             begin=begin,
@@ -187,7 +187,9 @@ def verify(
     max_chunks_syncs = mrsm.get_config('pipes', 'verify', 'max_chunks_syncs')
 
     info(
-        f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk"
+        f"Verifying {self}:\n    "
+        + ("Syncing" if not check_rowcounts_only else "Checking")
+        + f" {len(chunk_bounds)} chunk"
         + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
         + f" of size '{interval_str(chunk_interval)}'"
@@ -293,6 +295,17 @@ def verify(
         _batch_begin = batch_chunk_bounds[0][0]
         _batch_end = batch_chunk_bounds[-1][-1]
         batch_message_header = f"{_batch_begin} - {_batch_end}"
+
+        if check_rowcounts_only:
+            info(f"Checking row-counts for batch bounds:\n    {batch_message_header}")
+            _, (batch_init_success, batch_init_msg) = process_chunk_bounds(
+                (_batch_begin, _batch_end)
+            )
+            mrsm.pprint((batch_init_success, batch_init_msg))
+            if batch_init_success and 'up-to-date' in batch_init_msg:
+                info("Entire batch is up-to-date.")
+                return batch_init_success, batch_init_msg
+
         batch_bounds_success_tuples = dict(pool.map(process_chunk_bounds, batch_chunk_bounds))
         bounds_success_tuples.update(batch_bounds_success_tuples)
         batch_bounds_success_bools = {
@@ -404,7 +417,7 @@ def verify(
             retry_failed_batch = False
 
         batch_msg_to_print = (
-            f"{make_header('Completed batch ' + batch_counter_str + ' ' + for_self + ':')}\n{batch_msg}"
+            f"{make_header('Completed batch ' + batch_counter_str + ':')}\n{batch_msg}"
         )
         mrsm.pprint((batch_success, batch_msg_to_print))
 
@@ -480,11 +493,13 @@ def get_chunks_success_message(
     header = (header + "\n") if header else ""
     stats_msg = items_str(
         (
-            ([f'inserted {num_inserted:,}'] if num_inserted else [])
-            + ([f'updated {num_updated:,}'] if num_updated else [])
-            + ([f'upserted {num_upserted:,}'] if num_upserted else [])
-            + ([f'checked {num_checked:,}'] if num_checked else [])
-        ) or ['synced 0'],
+            (
+                ([f'inserted {num_inserted:,}'] if num_inserted else [])
+                + ([f'updated {num_updated:,}'] if num_updated else [])
+                + ([f'upserted {num_upserted:,}'] if num_upserted else [])
+                + ([f'checked {num_checked:,}'] if num_checked else [])
+            ) or ['synced 0']
+        ),
         quotes=False,
         and_=False,
     )

meerschaum/jobs/systemd.py

@@ -154,7 +154,7 @@ class SystemdExecutor(Executor):
             STATIC_CONFIG['environment']['systemd_delete_job']: (
                 '1'
                 if job.delete_after_completion
-                else '0',
+                else '0'
             ),
         })
 

meerschaum/plugins/_Plugin.py

@@ -450,7 +450,7 @@ class Plugin:
                 success, msg = False, (
                     f"Failed to run post-install setup for plugin '{self}'." + '\n' +
                     f"Check `setup()` in '{self.__file__}' for more information " +
-                    f"(no error message provided)."
+                    "(no error message provided)."
                 )
             else:
                 success, msg = True, success_msg
@@ -458,7 +458,7 @@ class Plugin:
             success = True
             msg = (
                 f"Post-install for plugin '{self}' returned None. " +
-                f"Assuming plugin successfully installed."
+                "Assuming plugin successfully installed."
             )
             warn(msg)
         else:
@@ -469,7 +469,7 @@ class Plugin:
             )
 
         _ongoing_installations.remove(self.full_name)
-        module = self.module
+        _ = self.module
         return success, msg
 
 
@@ -716,13 +716,14 @@ class Plugin:
         return required
 
 
-    def get_required_plugins(self, debug: bool=False) -> List[meerschaum.plugins.Plugin]:
+    def get_required_plugins(self, debug: bool=False) -> List[mrsm.plugins.Plugin]:
         """
         Return a list of required Plugin objects.
         """
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
         from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum.connectors.parse import is_valid_connector_keys
         plugins = []
         _deps = self.get_dependencies(debug=debug)
         sep = STATIC_CONFIG['plugins']['repo_separator']
@@ -731,11 +732,13 @@ class Plugin:
             if _d.startswith('plugin:') and len(_d) > len('plugin:')
         ]
         default_repo_keys = get_config('meerschaum', 'default_repository')
+        skipped_repo_keys = set()
+
         for _plugin_name in plugin_names:
             if sep in _plugin_name:
                 try:
                     _plugin_name, _repo_keys = _plugin_name.split(sep)
-                except Exception as e:
+                except Exception:
                     _repo_keys = default_repo_keys
                     warn(
                         f"Invalid repo keys for required plugin '{_plugin_name}'.\n    "
@@ -744,7 +747,20 @@ class Plugin:
                     )
             else:
                 _repo_keys = default_repo_keys
+
+            if _repo_keys in skipped_repo_keys:
+                continue
+
+            if not is_valid_connector_keys(_repo_keys):
+                warn(
+                    f"Invalid connector '{_repo_keys}'.\n"
+                    f"    Skipping required plugins from repository '{_repo_keys}'",
+                    stack=False,
+                )
+                continue
+
             plugins.append(Plugin(_plugin_name, repo=_repo_keys))
+
         return plugins
 
 

meerschaum/plugins/__init__.py

@@ -166,10 +166,11 @@ def post_sync_hook(
 
 _plugin_endpoints_to_pages = {}
 def web_page(
-        page: Union[str, None, Callable[[Any], Any]] = None,
-        login_required: bool = True,
-        **kwargs
-    ) -> Any:
+    page: Union[str, None, Callable[[Any], Any]] = None,
+    login_required: bool = True,
+    skip_navbar: bool = False,
+    **kwargs
+) -> Any:
     """
     Quickly add pages to the dash application.
 
@@ -200,6 +201,7 @@ def web_page(
         _plugin_endpoints_to_pages[page_str] = {
             'function': _func,
             'login_required': login_required,
+            'skip_navbar': skip_navbar,
         }
         return wrapper
 

meerschaum/utils/formatting/_shell.py

@@ -66,10 +66,7 @@ def clear_screen(debug: bool = False) -> bool:
             get_console().clear()
             print("", end="", flush=True)
             return True
-        clear_string = (
-            '\033c' if platform.system() != 'Windows'
-            else '\033[2J'
-        )
+        clear_string = '\033[2J'
         reset_string = '\033[0m'
         print(clear_string + reset_string, end="")
         print("", end="", flush=True)

meerschaum/utils/packages/_packages.py

@@ -58,7 +58,7 @@ packages: Dict[str, Dict[str, str]] = {
     '_internal'                      : {
         'apscheduler'                : (
                                        f"{_MRSM_PACKAGE_ARCHIVES_PREFIX}"
-                                       "APScheduler-4.0.0a5.post75+mrsm-py3-none-any.whl>=4.0.0a5"
+                                       "APScheduler-4.0.0a5.post87+mrsm-py3-none-any.whl>=4.0.0a5"
         ),
         'dataclass_wizard'           : 'dataclass-wizard>=0.28.0',
     },
@@ -135,6 +135,7 @@ packages: Dict[str, Dict[str, str]] = {
 packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
+    'geopandas'                      : 'geopandas>=1.0.1', 
     'pyarrow'                        : 'pyarrow>=16.1.0',
     'dask'                           : 'dask[complete]>=2024.12.1',
     'partd'                          : 'partd>=1.4.2',

meerschaum/utils/process.py

@@ -9,15 +9,25 @@ See `meerschaum.utils.pool` for multiprocessing and
 """
 
 from __future__ import annotations
-import os, signal, subprocess, sys, platform, traceback
+
+import os
+import signal
+import subprocess
+import sys
+import platform
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
 from meerschaum.config.static import STATIC_CONFIG
 
 _child_processes = []
 def signal_handler(sig, frame):
     for child in _child_processes:
-        child.send_signal(sig)
-        child.wait()
+        try:
+            child.send_signal(sig)
+            child.wait()
+        except Exception:
+            pass
 
 def run_process(
     *args,
@@ -223,3 +233,17 @@ def poll_process(
         watchdog_thread.cancel()
 
     return proc.poll()
+
+
+def _stop_process(
+    proc: subprocess.Popen,
+    timeout_seconds: int = 8,
+):
+    """
+    Stop a `subproccess.Popen` object.
+    """
+    proc.terminate()
+    try:
+        proc.wait(timeout=timeout_seconds)
+    except subprocess.TimeoutExpired:
+        proc.kill()

meerschaum/utils/schedule.py

@@ -95,7 +95,7 @@ def schedule_function(
     A `SuccessTuple` upon exit.
     """
     import asyncio
-    from meerschaum.utils.misc import filter_keywords, round_time
+    from meerschaum.utils.misc import filter_keywords
 
     global _scheduler
     kw['debug'] = debug
@@ -103,7 +103,7 @@ def schedule_function(
 
     _ = mrsm.attempt_import('attrs', lazy=False)
     apscheduler = mrsm.attempt_import('apscheduler', lazy=False)
-    now = round_time(datetime.now(timezone.utc), timedelta(minutes=1))
+    now = datetime.now(timezone.utc)
     trigger = parse_schedule(schedule, now=now)
     _scheduler = apscheduler.AsyncScheduler(identity='mrsm-scheduler')
     try:
@@ -296,7 +296,7 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
     starting_parts = schedule.split(STARTING_KEYWORD)
     starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()
-    now = now or round_time(datetime.now(timezone.utc), timedelta(minutes=1))
+    now = now or datetime.now(timezone.utc)
     try:
         if starting_str == 'now':
             starting_ts = now

meerschaum/utils/sql.py

@@ -1548,7 +1548,7 @@ def get_update_queries(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
-    flavor = flavor or (connectable.flavor if isinstance(connectable, SQLConnector) else None)
+    flavor = flavor or getattr(connectable, 'flavor', None)
     if not flavor:
         raise ValueError("Provide a flavor if using a SQLAlchemy session.")
     if (

meerschaum/utils/venv/__init__.py

@@ -410,6 +410,8 @@ def init_venv(
             pass
 
     def wait_for_lock():
+        if platform.system() == 'Windows':
+            return
         max_lock_seconds = 30.0
         sleep_message_seconds = 5.0
         step_sleep_seconds = 0.1
@@ -595,7 +597,7 @@ def venv_exec(
     as_proc: bool = False,
     capture_output: bool = True,
     debug: bool = False,
-) -> Union[bool, Tuple[int, bytes, bytes]]:
+) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
     """
     Execute Python code in a subprocess via a virtual environment's interpeter.
     Return `True` if the code successfully executes, `False` on failure.
@@ -630,6 +632,8 @@ def venv_exec(
     import subprocess
     import platform
     from meerschaum.utils.debug import dprint
+    from meerschaum.utils.process import _child_processes
+
     executable = venv_executable(venv=venv)
     cmd_list = [executable, '-c', code]
     if env is None:
@@ -656,6 +660,7 @@ def venv_exec(
         **group_kwargs
     )
     if as_proc:
+        _child_processes.append(process)
         return process
     stdout, stderr = process.communicate()
     exit_code = process.returncode

{meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.8.3
+Version: 2.9.0.dev1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -126,6 +126,7 @@ Requires-Dist: importlib-metadata>=4.12.0; extra == "extras"
 Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
+Requires-Dist: geopandas>=1.0.1; extra == "sql"
 Requires-Dist: pyarrow>=16.1.0; extra == "sql"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "sql"
 Requires-Dist: partd>=1.4.2; extra == "sql"
@@ -186,6 +187,7 @@ Requires-Dist: httpcore>=1.0.6; extra == "api"
 Requires-Dist: valkey>=6.0.0; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
+Requires-Dist: geopandas>=1.0.1; extra == "api"
 Requires-Dist: pyarrow>=16.1.0; extra == "api"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "api"
 Requires-Dist: partd>=1.4.2; extra == "api"
@@ -291,6 +293,7 @@ Requires-Dist: pywebview>=3.6.3; extra == "full"
 Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
+Requires-Dist: geopandas>=1.0.1; extra == "full"
 Requires-Dist: pyarrow>=16.1.0; extra == "full"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "full"
 Requires-Dist: partd>=1.4.2; extra == "full"