meerschaum 2.2.0rc2__tar.gz → 2.2.0rc3__tar.gz

{meerschaum-2.2.0rc2/meerschaum.egg-info → meerschaum-2.2.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.2.0rc2
+Version: 2.2.0rc3
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -71,7 +71,7 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "drivers"
 Requires-Dist: PyMySQL>=0.9.0; extra == "drivers"
 Requires-Dist: aiomysql>=0.0.21; extra == "drivers"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "drivers"
-Requires-Dist: duckdb>=0.9.0; extra == "drivers"
+Requires-Dist: duckdb<0.10.0; extra == "drivers"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "drivers"
 Provides-Extra: drivers
 Requires-Dist: pyodbc>=4.0.30; extra == "drivers"
@@ -120,8 +120,8 @@ Requires-Dist: importlib-metadata>=4.12.0; extra == "extras"
 Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
-Requires-Dist: pyarrow>=7.0.0; extra == "sql"
-Requires-Dist: dask>=2023.5.0; extra == "sql"
+Requires-Dist: pyarrow>=16.1.0; extra == "sql"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
@@ -133,7 +133,7 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "sql"
 Requires-Dist: PyMySQL>=0.9.0; extra == "sql"
 Requires-Dist: aiomysql>=0.0.21; extra == "sql"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "sql"
-Requires-Dist: duckdb>=0.9.0; extra == "sql"
+Requires-Dist: duckdb<0.10.0; extra == "sql"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "sql"
 Requires-Dist: wheel>=0.34.2; extra == "sql"
 Requires-Dist: setuptools>=63.3.0; extra == "sql"
@@ -167,19 +167,18 @@ Requires-Dist: dash-daq>=0.5.0; extra == "dash"
 Requires-Dist: terminado>=0.12.1; extra == "dash"
 Requires-Dist: tornado>=6.1.0; extra == "dash"
 Provides-Extra: api
-Requires-Dist: uvicorn[standard]>=0.22.0; extra == "api"
-Requires-Dist: gunicorn>=20.1.0; extra == "api"
+Requires-Dist: uvicorn[standard]>=0.29.0; extra == "api"
+Requires-Dist: gunicorn>=22.0.0; extra == "api"
 Requires-Dist: python-dotenv>=0.20.0; extra == "api"
 Requires-Dist: websockets>=11.0.3; extra == "api"
-Requires-Dist: fastapi>=0.100.0; extra == "api"
-Requires-Dist: passlib>=1.7.4; extra == "api"
+Requires-Dist: fastapi>=0.111.0; extra == "api"
 Requires-Dist: fastapi-login>=1.7.2; extra == "api"
-Requires-Dist: python-multipart>=0.0.5; extra == "api"
+Requires-Dist: python-multipart>=0.0.9; extra == "api"
 Requires-Dist: httpx>=0.24.1; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
-Requires-Dist: pyarrow>=7.0.0; extra == "api"
-Requires-Dist: dask>=2023.5.0; extra == "api"
+Requires-Dist: pyarrow>=16.1.0; extra == "api"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
@@ -191,7 +190,7 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "api"
 Requires-Dist: PyMySQL>=0.9.0; extra == "api"
 Requires-Dist: aiomysql>=0.0.21; extra == "api"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "api"
-Requires-Dist: duckdb>=0.9.0; extra == "api"
+Requires-Dist: duckdb<0.10.0; extra == "api"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "api"
 Requires-Dist: wheel>=0.34.2; extra == "api"
 Requires-Dist: setuptools>=63.3.0; extra == "api"
@@ -267,15 +266,15 @@ Requires-Dist: psycopg[binary]>=3.1.18; extra == "full"
 Requires-Dist: PyMySQL>=0.9.0; extra == "full"
 Requires-Dist: aiomysql>=0.0.21; extra == "full"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "full"
-Requires-Dist: duckdb>=0.9.0; extra == "full"
+Requires-Dist: duckdb<0.10.0; extra == "full"
 Requires-Dist: duckdb-engine>=0.9.2; extra == "full"
 Requires-Dist: toga>=0.3.0-dev29; extra == "full"
 Requires-Dist: pywebview>=3.6.3; extra == "full"
 Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
-Requires-Dist: pyarrow>=7.0.0; extra == "full"
-Requires-Dist: dask>=2023.5.0; extra == "full"
+Requires-Dist: pyarrow>=16.1.0; extra == "full"
+Requires-Dist: dask[dataframe]>=2024.5.1; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"
@@ -290,14 +289,13 @@ Requires-Dist: dash-extensions>=1.0.4; extra == "full"
 Requires-Dist: dash-daq>=0.5.0; extra == "full"
 Requires-Dist: terminado>=0.12.1; extra == "full"
 Requires-Dist: tornado>=6.1.0; extra == "full"
-Requires-Dist: uvicorn[standard]>=0.22.0; extra == "full"
-Requires-Dist: gunicorn>=20.1.0; extra == "full"
+Requires-Dist: uvicorn[standard]>=0.29.0; extra == "full"
+Requires-Dist: gunicorn>=22.0.0; extra == "full"
 Requires-Dist: python-dotenv>=0.20.0; extra == "full"
 Requires-Dist: websockets>=11.0.3; extra == "full"
-Requires-Dist: fastapi>=0.100.0; extra == "full"
-Requires-Dist: passlib>=1.7.4; extra == "full"
+Requires-Dist: fastapi>=0.111.0; extra == "full"
 Requires-Dist: fastapi-login>=1.7.2; extra == "full"
-Requires-Dist: python-multipart>=0.0.5; extra == "full"
+Requires-Dist: python-multipart>=0.0.9; extra == "full"
 Requires-Dist: httpx>=0.24.1; extra == "full"
 
 <img src="https://meerschaum.io/assets/banner_1920x320.png" alt="Meerschaum banner" style="width: 100%"/>

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/api/__init__.py

@@ -30,21 +30,26 @@ _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
 CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) != 'docker'
 
 endpoints = STATIC_CONFIG['api']['endpoints']
-aiofiles = attempt_import('aiofiles', lazy=False, check_update=CHECK_UPDATE)
+
+(
+    fastapi,
+    aiofiles,
+    starlette_responses,
+    multipart,
+    packaging_version,
+) = attempt_import(
+    'fastapi',
+    'aiofiles',
+    'starlette.responses',
+    'multipart',
+    'packaging.version',
+    lazy = False,
+    check_update = CHECK_UPDATE,
+)
 typing_extensions = attempt_import(
     'typing_extensions', lazy=False, check_update=CHECK_UPDATE,
     venv = None,
 )
-pydantic_dataclasses = attempt_import(
-    'pydantic.dataclasses', lazy=False, check_update=CHECK_UPDATE,
-)
-fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
-starlette_reponses = attempt_import(
-    'starlette.responses', warn=False, lazy=False,
-    check_update=CHECK_UPDATE,
-)
-python_multipart = attempt_import('multipart', lazy=False, check_update=CHECK_UPDATE)
-packaging_version = attempt_import('packaging.version', check_update=CHECK_UPDATE)
 from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
 

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/config/_jobs.py

@@ -18,7 +18,7 @@ default_jobs_config = {
         'max_file_size': 100_000,
         'lines_to_show': 30,
         'refresh_files_seconds': 5,
-        'min_buffer_len': 10,
+        'min_buffer_len': 5,
         'timestamp_format': '%Y-%m-%d %H:%M',
         'follow_timestamp_format': '%H:%M',
         'colors': [

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/config/_paths.py

@@ -129,6 +129,7 @@ paths = {
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
+    'PLUGINS_PACKAGES_INTERNAL_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'packaged_plugins'),
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.0rc2"
+__version__ = "2.2.0rc3"

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/config/static/__init__.py

@@ -60,6 +60,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
+        'daemon_id': 'MRSM_DAEMON_ID',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/__init__.py

@@ -317,6 +317,8 @@ def load_plugin_connectors():
     from meerschaum.plugins import get_plugins, import_plugins
     to_import = []
     for plugin in get_plugins():
+        if plugin is None:
+            continue
         with open(plugin.__file__, encoding='utf-8') as f:
             text = f.read()
         if 'make_connector' in text:

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/sql/SQLConnector.py

@@ -128,8 +128,10 @@ class SQLConnector(Connector):
         """
         if 'uri' in kw:
             uri = kw['uri']
-            if uri.startswith('postgres://'):
-                uri = uri.replace('postgres://', 'postgresql://', 1)
+            if uri.startswith('postgres') and not uri.startswith('postgresql'):
+                uri = uri.replace('postgres', 'postgresql', 1)
+            if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
+                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql://', 1)
                 flavor = 'timescaledb'

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/sql/_create_engine.py

@@ -28,7 +28,7 @@ default_create_engine_args = {
 }
 flavor_configs = {
     'timescaledb'      : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -38,7 +38,7 @@ flavor_configs = {
         },
     },
     'postgresql'         : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -48,7 +48,7 @@ flavor_configs = {
         },
     },
     'citus'            : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -242,7 +242,7 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in ('timescaledb',) and self.flavor in _uri:
-            engine_str = engine_str.replace(f'{self.flavor}://', 'postgresql://')
+            engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
 
     if debug:
         dprint(

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/sql/_instance.py

@@ -155,7 +155,9 @@ def _drop_old_temporary_tables(
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()
-    if refresh or not last_check or (now_ts - last_check) > 60:
+    if not last_check:
+        self._stale_temporary_tables_check_timestamp = 0
+    if refresh or (now_ts - last_check) < 60:
         self._stale_temporary_tables_check_timestamp = now_ts
         return self._drop_temporary_tables(debug=debug)
 

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/sql/_pipes.py

@@ -752,7 +752,7 @@ def get_pipe_data(
         debug = debug,
         **kw
     )
-
+    
     if is_dask:
         index_col = pipe.columns.get('datetime', None)
         kw['index_col'] = index_col
@@ -1478,43 +1478,11 @@ def sync_pipe_inplace(
     from meerschaum.utils.misc import generate_password
     from meerschaum.utils.debug import dprint
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
-    metadef = self.get_pipe_metadef(
-        pipe,
-        params = params,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
-    )
-    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
-    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
-    internal_schema = self.internal_schema
-    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
-
-    if not pipe.exists(debug=debug):
-        create_pipe_query = get_create_table_query(
-            metadef,
-            pipe.target,
-            self.flavor,
-            schema = self.get_pipe_schema(pipe),
-        )
-        result = self.exec(create_pipe_query, debug=debug)
-        if result is None:
-            return False, f"Could not insert new data into {pipe} from its SQL query definition."
-        if not self.create_indices(pipe, debug=debug):
-            warn(f"Failed to create indices for {pipe}. Continuing...")
-
-        rowcount = pipe.get_rowcount(debug=debug)
-        return True, f"Inserted {rowcount}, updated 0 rows."
-
-    session = sqlalchemy_orm.Session(self.engine)
-    connectable = session if self.flavor != 'duckdb' else self
-
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
         return '-' + transact_id + '_' + label + '_' + pipe.target
 
+    internal_schema = self.internal_schema
     temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
     temp_tables = {
         table_root: get_temp_table_name(table_root)
@@ -1528,6 +1496,17 @@ def sync_pipe_inplace(
         )
         for table_root, table_name_raw in temp_tables.items()
     }
+    metadef = self.get_pipe_metadef(
+        pipe,
+        params = params,
+        begin = begin,
+        end = end,
+        check_existing = check_existing,
+        debug = debug,
+    )
+    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
+    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
 
     def clean_up_temp_tables(ready_to_drop: bool = False):
         log_success, log_msg = self._log_temporary_tables_creation(
@@ -1541,6 +1520,36 @@ def sync_pipe_inplace(
         )
         if not log_success:
             warn(log_msg)
+        drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(
+            refresh = False,
+            debug = debug,
+        )
+        if not drop_stale_success:
+            warn(drop_stale_msg)
+        return drop_stale_success, drop_stale_msg
+
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    if not pipe.exists(debug=debug):
+        create_pipe_query = get_create_table_query(
+            metadef,
+            pipe.target,
+            self.flavor,
+            schema = self.get_pipe_schema(pipe),
+        )
+        result = self.exec(create_pipe_query, debug=debug)
+        if result is None:
+            _ = clean_up_temp_tables()
+            return False, f"Could not insert new data into {pipe} from its SQL query definition."
+
+        if not self.create_indices(pipe, debug=debug):
+            warn(f"Failed to create indices for {pipe}. Continuing...")
+
+        rowcount = pipe.get_rowcount(debug=debug)
+        _ = clean_up_temp_tables()
+        return True, f"Inserted {rowcount}, updated 0 rows."
+
+    session = sqlalchemy_orm.Session(self.engine)
+    connectable = session if self.flavor != 'duckdb' else self
 
     create_new_query = get_create_table_query(
         metadef,
@@ -1908,10 +1917,6 @@ def sync_pipe_inplace(
     )
     _ = clean_up_temp_tables(ready_to_drop=True)
 
-    drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(refresh=False, debug=debug)
-    if not drop_stale_success:
-        warn(drop_stale_msg)
-
     return True, msg
 
 
@@ -2372,6 +2377,16 @@ def get_pipe_columns_types(
     """
     if not pipe.exists(debug=debug):
         return {}
+
+    if self.flavor == 'duckdb':
+        from meerschaum.utils.sql import get_table_cols_types
+        return get_table_cols_types(
+            pipe.target,
+            self,
+            flavor = self.flavor,
+            schema = self.schema,
+        )
+
     table_columns = {}
     try:
         pipe_table = self.get_pipe_table(pipe, debug=debug)

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/connectors/sql/_sql.py

@@ -943,17 +943,15 @@ def psql_insert_copy(
         ) for row in data_iter
     )
 
+    table_name = sql_item_name(table.name, 'postgresql', table.schema)
+    columns = ', '.join(f'"{k}"' for k in keys)
+    sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
-        s_buf = StringIO()
-        writer = csv.writer(s_buf)
-        writer.writerows(data_iter)
-        s_buf.seek(0)
-
-        columns = ', '.join(f'"{k}"' for k in keys)
-        table_name = sql_item_name(table.name, 'postgresql', table.schema)
-        sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
-        cur.copy_expert(sql=sql, file=s_buf)
+        with cur.copy(sql) as copy:
+            writer = csv.writer(copy)
+            writer.writerows(data_iter)
 
 
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/core/User/_User.py

@@ -86,6 +86,8 @@ def verify_password(
     -------
     A `bool` indicating whether `password` matches `password_hash`.
     """
+    if password is None or password_hash is None:
+        return False
     hash_config = STATIC_CONFIG['users']['password_hash']
     try:
         digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/plugins/__init__.py

@@ -247,6 +247,26 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 _warn(f"Unable to create lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")
 
     with _locks['internal_plugins']:
+
+        try:
+            from importlib.metadata import entry_points
+        except ImportError:
+            importlib_metadata = attempt_import('importlib_metadata', lazy=False)
+            entry_points = importlib_metadata.entry_points
+
+        ### NOTE: Allow plugins to be installed via `pip`.
+        packaged_plugin_paths = []
+        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        for ep in discovered_packaged_plugins_eps:
+            module_name = ep.name
+            for package_file_path in ep.dist.files:
+                if package_file_path.suffix != '.py':
+                    continue
+                if str(package_file_path) == f'{module_name}.py':
+                    packaged_plugin_paths.append(package_file_path.locate())
+                elif str(package_file_path) == f'{module_name}/__init__.py':
+                    packaged_plugin_paths.append(package_file_path.locate().parent)
+
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
@@ -255,7 +275,6 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
 
-
         existing_symlinked_paths = [
             (PLUGINS_RESOURCES_PATH / item) 
             for item in os.listdir(PLUGINS_RESOURCES_PATH)
@@ -275,6 +294,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 for plugins_path in PLUGINS_DIR_PATHS
             ]
         ))
+        plugins_to_be_symlinked.extend(packaged_plugin_paths)
 
         ### Check for duplicates.
         seen_plugins = defaultdict(lambda: 0)
@@ -538,6 +558,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
     ]
     plugins = tuple(plugin for plugin in _plugins if plugin.is_installed(try_import=try_import))
     if len(to_load) == 1:
+        if len(plugins) == 0:
+            raise ValueError(f"Plugin '{to_load[0]}' is not installed.")
         return plugins[0]
     return plugins
 

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/utils/daemon/Daemon.py

@@ -19,6 +19,7 @@ from functools import partial
 from datetime import datetime, timezone
 from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
 from meerschaum.config import get_config
+from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH
 from meerschaum.config._patch import apply_patch_to_config
 from meerschaum.utils.warnings import warn, error
@@ -170,9 +171,11 @@ class Daemon:
             log_refresh_seconds,
             partial(self.rotating_log.refresh_files, start_interception=True),
         )
+
         try:
             os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
             with self._daemon_context:
+                os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
                 self.rotating_log.refresh_files(start_interception=True)
                 try:
                     with open(self.pid_path, 'w+', encoding='utf-8') as f:
@@ -462,6 +465,9 @@ class Daemon:
         Handle `SIGINT` within the Daemon context.
         This method is injected into the `DaemonContext`.
         """
+        #  from meerschaum.utils.daemon.FileDescriptorInterceptor import STOP_READING_FD_EVENT
+        #  STOP_READING_FD_EVENT.set()
+        self.rotating_log.stop_log_fd_interception(unused_only=False)
         timer = self.__dict__.get('_log_refresh_timer', None)
         if timer is not None:
             timer.cancel()
@@ -471,7 +477,16 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-        self.rotating_log.stop_log_fd_interception()
+        import threading
+        for thread in threading.enumerate():
+            if thread.name == 'MainThread':
+                continue
+            try:
+                if thread.is_alive():
+                    stack = traceback.format_stack(sys._current_frames()[thread.ident])
+                    thread.join()
+            except Exception as e:
+                warn(traceback.format_exc())
         raise KeyboardInterrupt()
 
 
@@ -489,7 +504,7 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-        raise SystemExit(1)
+        raise SystemExit(0)
 
  
     def _send_signal(

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/utils/daemon/FileDescriptorInterceptor.py

@@ -7,12 +7,15 @@ Intercept OS-level file descriptors.
 """
 
 import os
+import select
 import traceback
+from threading import Event
 from datetime import datetime
 from meerschaum.utils.typing import Callable
 from meerschaum.utils.warnings import warn
 
 FD_CLOSED: int = 9
+STOP_READING_FD_EVENT: Event = Event()
 
 class FileDescriptorInterceptor:
     """
@@ -32,10 +35,12 @@ class FileDescriptorInterceptor:
         injection_hook: Callable[[], str]
             A callable which returns a string to be injected into the written data.
         """
+        self.stop_event = Event()
         self.injection_hook = injection_hook
         self.original_file_descriptor = file_descriptor
         self.new_file_descriptor = os.dup(file_descriptor)
         self.read_pipe, self.write_pipe = os.pipe()
+        self.signal_read_pipe, self.signal_write_pipe = os.pipe()
         os.dup2(self.write_pipe, file_descriptor)
 
     def start_interception(self):
@@ -44,11 +49,23 @@ class FileDescriptorInterceptor:
 
         NOTE: This is blocking and is meant to be run in a thread.
         """
+        os.set_blocking(self.read_pipe, False)
+        os.set_blocking(self.signal_read_pipe, False)
         is_first_read = True
-        while True:
-            data = os.read(self.read_pipe, 1024)
-            if not data:
-                break
+        while not self.stop_event.is_set():
+            try:
+                rlist, _, _ = select.select([self.read_pipe, self.signal_read_pipe], [], [], 0.1)
+                if self.signal_read_pipe in rlist:
+                    break
+                if not rlist:
+                    continue
+                data = os.read(self.read_pipe, 1024)
+                if not data:
+                    break
+            except BlockingIOError:
+                continue
+            except OSError as e:
+                continue
 
             first_char_is_newline = data[0] == b'\n'
             last_char_is_newline = data[-1] == b'\n'
@@ -65,16 +82,17 @@ class FileDescriptorInterceptor:
                 if last_char_is_newline
                 else data.replace(b'\n', b'\n' + injected_bytes)
             )
-
             os.write(self.new_file_descriptor, modified_data)
 
+
     def stop_interception(self):
         """
-        Restore the file descriptors and close the new pipes.
+        Close the new file descriptors.
         """
+        self.stop_event.set()
+        os.write(self.signal_write_pipe, b'\0')
         try:
-            os.dup2(self.new_file_descriptor, self.original_file_descriptor)
-            #  os.close(self.new_file_descriptor)
+            os.close(self.new_file_descriptor)
         except OSError as e:
             if e.errno != FD_CLOSED:
                 warn(
@@ -100,3 +118,23 @@ class FileDescriptorInterceptor:
                     + "to the intercepted file descriptor:\n"
                     + f"{traceback.format_exc()}"
                 )
+
+        try:
+            os.close(self.signal_read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.signal_write_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/utils/daemon/RotatingFile.py

@@ -621,6 +621,10 @@ class RotatingFile(io.IOBase):
             self._stdout_interceptor_thread,
             self._stderr_interceptor_thread,
         ])
+        self._interceptors.extend([
+            self._stdout_interceptor,
+            self._stderr_interceptor,
+        ])
         self.stop_log_fd_interception(unused_only=True)
 
     def stop_log_fd_interception(self, unused_only: bool = False):

{meerschaum-2.2.0rc2 → meerschaum-2.2.0rc3}/meerschaum/utils/daemon/__init__.py

@@ -64,6 +64,8 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
         ### Only run if the kwargs equal or no actions are provided.
         if existing_kwargs == _args or not _args.get('action', []):
+            if daemon.status == 'running':
+                return True, f"Daemon '{daemon}' is already running."
             return daemon.run(
                 debug = debug,
                 allow_dirty_run = True,