meerschaum 2.2.0rc2__tar.gz → 2.2.0rc4__tar.gz

{meerschaum-2.2.0rc2/meerschaum.egg-info → meerschaum-2.2.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.2.0rc2
+Version: 2.2.0rc4
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -42,40 +42,40 @@ Requires-Dist: colorama>=0.4.3; extra == "formatting"
 Requires-Dist: rich>=13.4.2; extra == "formatting"
 Requires-Dist: more-termcolor>=1.1.3; extra == "formatting"
 Requires-Dist: humanfriendly>=10.0.0; extra == "formatting"
-Provides-Extra: required
-Requires-Dist: wheel>=0.34.2; extra == "required"
-Requires-Dist: setuptools>=63.3.0; extra == "required"
-Requires-Dist: PyYAML>=5.3.1; extra == "required"
-Requires-Dist: pip>=22.0.4; extra == "required"
-Requires-Dist: update-checker>=0.18.0; extra == "required"
-Requires-Dist: semver>=3.0.0; extra == "required"
-Requires-Dist: pathspec>=0.9.0; extra == "required"
-Requires-Dist: python-dateutil>=2.7.5; extra == "required"
-Requires-Dist: requests>=2.23.0; extra == "required"
-Requires-Dist: binaryornot>=0.4.4; extra == "required"
-Requires-Dist: pyvim>=3.0.2; extra == "required"
-Requires-Dist: aiofiles>=0.6.0; extra == "required"
-Requires-Dist: packaging>=21.3.0; extra == "required"
-Requires-Dist: prompt-toolkit>=3.0.39; extra == "required"
-Requires-Dist: more-itertools>=8.7.0; extra == "required"
-Requires-Dist: python-daemon>=0.2.3; extra == "required"
-Requires-Dist: fasteners>=0.18.0; extra == "required"
-Requires-Dist: psutil>=5.8.0; extra == "required"
-Requires-Dist: watchfiles>=0.21.0; extra == "required"
-Requires-Dist: dill>=0.3.3; extra == "required"
-Requires-Dist: virtualenv>=20.1.0; extra == "required"
-Requires-Dist: APScheduler>=4.0.0a5; extra == "required"
+Provides-Extra: core
+Requires-Dist: wheel>=0.34.2; extra == "core"
+Requires-Dist: setuptools>=63.3.0; extra == "core"
+Requires-Dist: PyYAML>=5.3.1; extra == "core"
+Requires-Dist: pip>=22.0.4; extra == "core"
+Requires-Dist: update-checker>=0.18.0; extra == "core"
+Requires-Dist: semver>=3.0.0; extra == "core"
+Requires-Dist: pathspec>=0.9.0; extra == "core"
+Requires-Dist: python-dateutil>=2.7.5; extra == "core"
+Requires-Dist: requests>=2.23.0; extra == "core"
+Requires-Dist: binaryornot>=0.4.4; extra == "core"
+Requires-Dist: pyvim>=3.0.2; extra == "core"
+Requires-Dist: aiofiles>=0.6.0; extra == "core"
+Requires-Dist: packaging>=21.3.0; extra == "core"
+Requires-Dist: prompt-toolkit>=3.0.39; extra == "core"
+Requires-Dist: more-itertools>=8.7.0; extra == "core"
+Requires-Dist: python-daemon>=0.2.3; extra == "core"
+Requires-Dist: fasteners>=0.18.0; extra == "core"
+Requires-Dist: psutil>=5.8.0; extra == "core"
+Requires-Dist: watchfiles>=0.21.0; extra == "core"
+Requires-Dist: dill>=0.3.3; extra == "core"
+Requires-Dist: virtualenv>=20.1.0; extra == "core"
+Requires-Dist: APScheduler>=4.0.0a5; extra == "core"
 Provides-Extra: drivers
 Requires-Dist: cryptography>=38.0.1; extra == "drivers"
 Requires-Dist: psycopg[binary]>=3.1.18; extra == "drivers"
 Requires-Dist: PyMySQL>=0.9.0; extra == "drivers"
 Requires-Dist: aiomysql>=0.0.21; extra == "drivers"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "drivers"
-Requires-Dist: duckdb>=0.9.0; extra == "drivers"
+Requires-Dist: duckdb<0.10.3; extra == "drivers"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "drivers"
-Provides-Extra: drivers
-Requires-Dist: pyodbc>=4.0.30; extra == "drivers"
-Requires-Dist: cx_Oracle>=8.3.0; extra == "drivers"
+Provides-Extra: drivers-extras
+Requires-Dist: pyodbc>=4.0.30; extra == "drivers-extras"
+Requires-Dist: cx_Oracle>=8.3.0; extra == "drivers-extras"
 Provides-Extra: cli
 Requires-Dist: pgcli>=3.1.0; extra == "cli"
 Requires-Dist: mycli>=1.23.2; extra == "cli"
@@ -120,8 +120,8 @@ Requires-Dist: importlib-metadata>=4.12.0; extra == "extras"
 Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
-Requires-Dist: pyarrow>=7.0.0; extra == "sql"
-Requires-Dist: dask>=2023.5.0; extra == "sql"
+Requires-Dist: pyarrow>=16.1.0; extra == "sql"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
@@ -133,7 +133,7 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "sql"
 Requires-Dist: PyMySQL>=0.9.0; extra == "sql"
 Requires-Dist: aiomysql>=0.0.21; extra == "sql"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "sql"
-Requires-Dist: duckdb>=0.9.0; extra == "sql"
+Requires-Dist: duckdb<0.10.3; extra == "sql"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "sql"
 Requires-Dist: wheel>=0.34.2; extra == "sql"
 Requires-Dist: setuptools>=63.3.0; extra == "sql"
@@ -167,19 +167,18 @@ Requires-Dist: dash-daq>=0.5.0; extra == "dash"
 Requires-Dist: terminado>=0.12.1; extra == "dash"
 Requires-Dist: tornado>=6.1.0; extra == "dash"
 Provides-Extra: api
-Requires-Dist: uvicorn[standard]>=0.22.0; extra == "api"
-Requires-Dist: gunicorn>=20.1.0; extra == "api"
+Requires-Dist: uvicorn[standard]>=0.29.0; extra == "api"
+Requires-Dist: gunicorn>=22.0.0; extra == "api"
 Requires-Dist: python-dotenv>=0.20.0; extra == "api"
 Requires-Dist: websockets>=11.0.3; extra == "api"
-Requires-Dist: fastapi>=0.100.0; extra == "api"
-Requires-Dist: passlib>=1.7.4; extra == "api"
+Requires-Dist: fastapi>=0.111.0; extra == "api"
 Requires-Dist: fastapi-login>=1.7.2; extra == "api"
-Requires-Dist: python-multipart>=0.0.5; extra == "api"
+Requires-Dist: python-multipart>=0.0.9; extra == "api"
 Requires-Dist: httpx>=0.24.1; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
-Requires-Dist: pyarrow>=7.0.0; extra == "api"
-Requires-Dist: dask>=2023.5.0; extra == "api"
+Requires-Dist: pyarrow>=16.1.0; extra == "api"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
@@ -191,7 +190,7 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "api"
 Requires-Dist: PyMySQL>=0.9.0; extra == "api"
 Requires-Dist: aiomysql>=0.0.21; extra == "api"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "api"
-Requires-Dist: duckdb>=0.9.0; extra == "api"
+Requires-Dist: duckdb<0.10.3; extra == "api"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "api"
 Requires-Dist: wheel>=0.34.2; extra == "api"
 Requires-Dist: setuptools>=63.3.0; extra == "api"
@@ -267,15 +266,15 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "full"
 Requires-Dist: PyMySQL>=0.9.0; extra == "full"
 Requires-Dist: aiomysql>=0.0.21; extra == "full"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "full"
-Requires-Dist: duckdb>=0.9.0; extra == "full"
+Requires-Dist: duckdb<0.10.3; extra == "full"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "full"
 Requires-Dist: toga>=0.3.0-dev29; extra == "full"
 Requires-Dist: pywebview>=3.6.3; extra == "full"
 Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
-Requires-Dist: pyarrow>=7.0.0; extra == "full"
-Requires-Dist: dask>=2023.5.0; extra == "full"
+Requires-Dist: pyarrow>=16.1.0; extra == "full"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"
@@ -290,14 +289,13 @@ Requires-Dist: dash-extensions>=1.0.4; extra == "full"
 Requires-Dist: dash-daq>=0.5.0; extra == "full"
 Requires-Dist: terminado>=0.12.1; extra == "full"
 Requires-Dist: tornado>=6.1.0; extra == "full"
-Requires-Dist: uvicorn[standard]>=0.22.0; extra == "full"
-Requires-Dist: gunicorn>=20.1.0; extra == "full"
+Requires-Dist: uvicorn[standard]>=0.29.0; extra == "full"
+Requires-Dist: gunicorn>=22.0.0; extra == "full"
 Requires-Dist: python-dotenv>=0.20.0; extra == "full"
 Requires-Dist: websockets>=11.0.3; extra == "full"
-Requires-Dist: fastapi>=0.100.0; extra == "full"
-Requires-Dist: passlib>=1.7.4; extra == "full"
+Requires-Dist: fastapi>=0.111.0; extra == "full"
 Requires-Dist: fastapi-login>=1.7.2; extra == "full"
-Requires-Dist: python-multipart>=0.0.5; extra == "full"
+Requires-Dist: python-multipart>=0.0.9; extra == "full"
 Requires-Dist: httpx>=0.24.1; extra == "full"
 
 <img src="https://meerschaum.io/assets/banner_1920x320.png" alt="Meerschaum banner" style="width: 100%"/>

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/actions/show.py

@@ -589,8 +589,8 @@ def _show_logs(
     if not ANSI:
         info = print
     colors = get_config('jobs', 'logs', 'colors')
-    timestamp_format = get_config('jobs', 'logs', 'timestamp_format')
-    follow_timestamp_format = get_config('jobs', 'logs', 'follow_timestamp_format')
+    timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
+    follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
     daemons = get_filtered_daemons(action)
     now = datetime.now(timezone.utc)
     now_str = now.strftime(timestamp_format)

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/api/__init__.py

@@ -30,25 +30,35 @@ _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
 CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) != 'docker'
 
 endpoints = STATIC_CONFIG['api']['endpoints']
-aiofiles = attempt_import('aiofiles', lazy=False, check_update=CHECK_UPDATE)
-typing_extensions = attempt_import(
-    'typing_extensions', lazy=False, check_update=CHECK_UPDATE,
-    venv = None,
-)
-pydantic_dataclasses = attempt_import(
-    'pydantic.dataclasses', lazy=False, check_update=CHECK_UPDATE,
+
+(
+    fastapi,
+    aiofiles,
+    starlette_responses,
+    multipart,
+    packaging_version,
+) = attempt_import(
+    'fastapi',
+    'aiofiles',
+    'starlette.responses',
+    'multipart',
+    'packaging.version',
+    lazy = False,
+    check_update = CHECK_UPDATE,
 )
-fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
-starlette_reponses = attempt_import(
-    'starlette.responses', warn=False, lazy=False,
-    check_update=CHECK_UPDATE,
+(
+    typing_extensions,
+    uvicorn_workers,
+) = attempt_import(
+    'typing_extensions',
+    'uvicorn.workers',
+    lazy = False,
+    check_update = CHECK_UPDATE,
+    venv = None,
 )
-python_multipart = attempt_import('multipart', lazy=False, check_update=CHECK_UPDATE)
-packaging_version = attempt_import('packaging.version', check_update=CHECK_UPDATE)
 from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
 
-uvicorn_workers = attempt_import('uvicorn.workers', venv=None, check_update=CHECK_UPDATE)
 uvicorn_config = None
 sys_config = get_config('system', 'api')
 permissions_config = get_config('system', 'api', 'permissions')

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/api/_oauth2.py

@@ -7,11 +7,11 @@ Define JWT authorization here.
 """
 
 import os
-from meerschaum.api import app, endpoints
+from meerschaum.api import app, endpoints, CHECK_UPDATE
 from meerschaum.utils.packages import attempt_import
-fastapi = attempt_import('fastapi', lazy=False)
-fastapi_responses = attempt_import('fastapi.responses', lazy=False)
-fastapi_login = attempt_import('fastapi_login')
+fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
+fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
+fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
 
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/api/dash/callbacks/dashboard.py

@@ -713,11 +713,11 @@ def download_pipe_csv(n_clicks):
     pipe = pipe_from_ctx(ctx, 'n_clicks')
     if pipe is None:
         raise PreventUpdate
-    filename = str(pipe.target) + '.csv'
     bounds = pipe.get_chunk_bounds(bounded=True, debug=debug)
-    begin, _ = bounds[-1]
+    begin, end = bounds[-1]
+    filename = str(pipe.target) + f" {begin} - {end}.csv"
     try:
-        df = pipe.get_data(begin=begin, end=None, debug=debug)
+        df = pipe.get_data(begin=begin, end=end, debug=debug)
     except Exception as e:
         df = None
     if df is not None:

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/api/dash/keys.py

@@ -108,7 +108,7 @@ action_dropdown_row = html.Div(
                             id = 'flags-dropdown',
                             multi = True,
                             placeholder = 'Boolean flags',
-                            options = [],
+                            options = ['--yes'],
                             value = ['--yes'],
                         ),
                         id = 'flags-dropdown-div',

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/api/dash/pages/dashboard.py

@@ -11,12 +11,22 @@ from meerschaum.config import __doc__ as doc, get_config
 from meerschaum.utils.misc import get_connector_labels
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc, import_pandas
 from meerschaum.api import endpoints, CHECK_UPDATE
-dex = attempt_import('dash_extensions', lazy=False, check_update=CHECK_UPDATE)
-dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
+(
+    dex,
+    px,
+    daq,
+    dbc,
+) = attempt_import(
+    'dash_extensions',
+    'plotly.express',
+    'dash_daq',
+    'dash_bootstrap_components',
+    lazy = False,
+    warn = False,
+    check_update = CHECK_UPDATE,
+)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 pd = import_pandas(check_update=CHECK_UPDATE)
-px = attempt_import('plotly.express', warn=False, check_update=CHECK_UPDATE)
-daq = attempt_import('dash_daq', warn=False, check_update=CHECK_UPDATE)
 
 from meerschaum.api.dash.components import (
     go_button,

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/config/_jobs.py

@@ -14,13 +14,16 @@ default_jobs_config = {
         'columns': 70,
     },
     'logs': {
+        'timestamps': {
+            'enabled': True,
+            'format': '%Y-%m-%d %H:%M',
+            'follow_format': '%H:%M',
+        },
         'num_files_to_keep': 5,
         'max_file_size': 100_000,
         'lines_to_show': 30,
         'refresh_files_seconds': 5,
-        'min_buffer_len': 10,
-        'timestamp_format': '%Y-%m-%d %H:%M',
-        'follow_timestamp_format': '%H:%M',
+        'min_buffer_len': 5,
         'colors': [
             'cyan',
             'magenta',

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/config/_paths.py

@@ -129,6 +129,7 @@ paths = {
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
+    'PLUGINS_PACKAGES_INTERNAL_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'packaged_plugins'),
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.0rc2"
+__version__ = "2.2.0rc4"

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/config/static/__init__.py

@@ -60,6 +60,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
+        'daemon_id': 'MRSM_DAEMON_ID',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/__init__.py

@@ -317,6 +317,8 @@ def load_plugin_connectors():
     from meerschaum.plugins import get_plugins, import_plugins
     to_import = []
     for plugin in get_plugins():
+        if plugin is None:
+            continue
         with open(plugin.__file__, encoding='utf-8') as f:
             text = f.read()
         if 'make_connector' in text:

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/sql/SQLConnector.py

@@ -128,8 +128,10 @@ class SQLConnector(Connector):
         """
         if 'uri' in kw:
             uri = kw['uri']
-            if uri.startswith('postgres://'):
-                uri = uri.replace('postgres://', 'postgresql://', 1)
+            if uri.startswith('postgres') and not uri.startswith('postgresql'):
+                uri = uri.replace('postgres', 'postgresql', 1)
+            if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
+                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql://', 1)
                 flavor = 'timescaledb'

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/sql/_create_engine.py

@@ -28,7 +28,7 @@ default_create_engine_args = {
 }
 flavor_configs = {
     'timescaledb'      : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -38,7 +38,7 @@ flavor_configs = {
         },
     },
     'postgresql'         : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -48,7 +48,7 @@ flavor_configs = {
         },
     },
     'citus'            : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -242,7 +242,7 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in ('timescaledb',) and self.flavor in _uri:
-            engine_str = engine_str.replace(f'{self.flavor}://', 'postgresql://')
+            engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
 
     if debug:
         dprint(

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/sql/_instance.py

@@ -155,7 +155,9 @@ def _drop_old_temporary_tables(
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()
-    if refresh or not last_check or (now_ts - last_check) > 60:
+    if not last_check:
+        self._stale_temporary_tables_check_timestamp = 0
+    if refresh or (now_ts - last_check) < 60:
         self._stale_temporary_tables_check_timestamp = now_ts
         return self._drop_temporary_tables(debug=debug)
 

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/sql/_pipes.py

@@ -752,7 +752,7 @@ def get_pipe_data(
         debug = debug,
         **kw
     )
-
+    
     if is_dask:
         index_col = pipe.columns.get('datetime', None)
         kw['index_col'] = index_col
@@ -763,6 +763,7 @@ def get_pipe_data(
         if typ == 'numeric' and col in dtypes
     ]
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
+    
     df = self.read(
         query,
         dtype = dtypes,
@@ -1478,43 +1479,11 @@ def sync_pipe_inplace(
     from meerschaum.utils.misc import generate_password
     from meerschaum.utils.debug import dprint
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
-    metadef = self.get_pipe_metadef(
-        pipe,
-        params = params,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
-    )
-    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
-    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
-    internal_schema = self.internal_schema
-    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
-
-    if not pipe.exists(debug=debug):
-        create_pipe_query = get_create_table_query(
-            metadef,
-            pipe.target,
-            self.flavor,
-            schema = self.get_pipe_schema(pipe),
-        )
-        result = self.exec(create_pipe_query, debug=debug)
-        if result is None:
-            return False, f"Could not insert new data into {pipe} from its SQL query definition."
-        if not self.create_indices(pipe, debug=debug):
-            warn(f"Failed to create indices for {pipe}. Continuing...")
-
-        rowcount = pipe.get_rowcount(debug=debug)
-        return True, f"Inserted {rowcount}, updated 0 rows."
-
-    session = sqlalchemy_orm.Session(self.engine)
-    connectable = session if self.flavor != 'duckdb' else self
-
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
         return '-' + transact_id + '_' + label + '_' + pipe.target
 
+    internal_schema = self.internal_schema
     temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
     temp_tables = {
         table_root: get_temp_table_name(table_root)
@@ -1528,6 +1497,17 @@ def sync_pipe_inplace(
         )
         for table_root, table_name_raw in temp_tables.items()
     }
+    metadef = self.get_pipe_metadef(
+        pipe,
+        params = params,
+        begin = begin,
+        end = end,
+        check_existing = check_existing,
+        debug = debug,
+    )
+    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
+    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
 
     def clean_up_temp_tables(ready_to_drop: bool = False):
         log_success, log_msg = self._log_temporary_tables_creation(
@@ -1541,6 +1521,36 @@ def sync_pipe_inplace(
         )
         if not log_success:
             warn(log_msg)
+        drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(
+            refresh = False,
+            debug = debug,
+        )
+        if not drop_stale_success:
+            warn(drop_stale_msg)
+        return drop_stale_success, drop_stale_msg
+
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    if not pipe.exists(debug=debug):
+        create_pipe_query = get_create_table_query(
+            metadef,
+            pipe.target,
+            self.flavor,
+            schema = self.get_pipe_schema(pipe),
+        )
+        result = self.exec(create_pipe_query, debug=debug)
+        if result is None:
+            _ = clean_up_temp_tables()
+            return False, f"Could not insert new data into {pipe} from its SQL query definition."
+
+        if not self.create_indices(pipe, debug=debug):
+            warn(f"Failed to create indices for {pipe}. Continuing...")
+
+        rowcount = pipe.get_rowcount(debug=debug)
+        _ = clean_up_temp_tables()
+        return True, f"Inserted {rowcount}, updated 0 rows."
+
+    session = sqlalchemy_orm.Session(self.engine)
+    connectable = session if self.flavor != 'duckdb' else self
 
     create_new_query = get_create_table_query(
         metadef,
@@ -1908,10 +1918,6 @@ def sync_pipe_inplace(
     )
     _ = clean_up_temp_tables(ready_to_drop=True)
 
-    drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(refresh=False, debug=debug)
-    if not drop_stale_success:
-        warn(drop_stale_msg)
-
     return True, msg
 
 
@@ -2372,6 +2378,16 @@ def get_pipe_columns_types(
     """
     if not pipe.exists(debug=debug):
         return {}
+
+    if self.flavor == 'duckdb':
+        from meerschaum.utils.sql import get_table_cols_types
+        return get_table_cols_types(
+            pipe.target,
+            self,
+            flavor = self.flavor,
+            schema = self.schema,
+        )
+
     table_columns = {}
     try:
         pipe_table = self.get_pipe_table(pipe, debug=debug)

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/connectors/sql/_sql.py

@@ -943,17 +943,15 @@ def psql_insert_copy(
         ) for row in data_iter
     )
 
+    table_name = sql_item_name(table.name, 'postgresql', table.schema)
+    columns = ', '.join(f'"{k}"' for k in keys)
+    sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
-        s_buf = StringIO()
-        writer = csv.writer(s_buf)
-        writer.writerows(data_iter)
-        s_buf.seek(0)
-
-        columns = ', '.join(f'"{k}"' for k in keys)
-        table_name = sql_item_name(table.name, 'postgresql', table.schema)
-        sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
-        cur.copy_expert(sql=sql, file=s_buf)
+        with cur.copy(sql) as copy:
+            writer = csv.writer(copy)
+            writer.writerows(data_iter)
 
 
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/core/User/_User.py

@@ -86,6 +86,8 @@ def verify_password(
     -------
     A `bool` indicating whether `password` matches `password_hash`.
     """
+    if password is None or password_hash is None:
+        return False
     hash_config = STATIC_CONFIG['users']['password_hash']
     try:
         digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc4}/meerschaum/plugins/__init__.py

@@ -247,6 +247,26 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 _warn(f"Unable to create lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")
 
     with _locks['internal_plugins']:
+
+        try:
+            from importlib.metadata import entry_points
+        except ImportError:
+            importlib_metadata = attempt_import('importlib_metadata', lazy=False)
+            entry_points = importlib_metadata.entry_points
+
+        ### NOTE: Allow plugins to be installed via `pip`.
+        packaged_plugin_paths = []
+        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        for ep in discovered_packaged_plugins_eps:
+            module_name = ep.name
+            for package_file_path in ep.dist.files:
+                if package_file_path.suffix != '.py':
+                    continue
+                if str(package_file_path) == f'{module_name}.py':
+                    packaged_plugin_paths.append(package_file_path.locate())
+                elif str(package_file_path) == f'{module_name}/__init__.py':
+                    packaged_plugin_paths.append(package_file_path.locate().parent)
+
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
@@ -255,7 +275,6 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
 
-
         existing_symlinked_paths = [
             (PLUGINS_RESOURCES_PATH / item) 
             for item in os.listdir(PLUGINS_RESOURCES_PATH)
@@ -275,6 +294,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 for plugins_path in PLUGINS_DIR_PATHS
             ]
         ))
+        plugins_to_be_symlinked.extend(packaged_plugin_paths)
 
         ### Check for duplicates.
         seen_plugins = defaultdict(lambda: 0)
@@ -538,6 +558,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
     ]
     plugins = tuple(plugin for plugin in _plugins if plugin.is_installed(try_import=try_import))
     if len(to_load) == 1:
+        if len(plugins) == 0:
+            raise ValueError(f"Plugin '{to_load[0]}' is not installed.")
         return plugins[0]
     return plugins