meerschaum 2.2.0rc1__py3-none-any.whl → 2.2.0rc3__py3-none-any.whl

meerschaum/connectors/sql/SQLConnector.py

@@ -128,8 +128,10 @@ class SQLConnector(Connector):
         """
         if 'uri' in kw:
             uri = kw['uri']
-            if uri.startswith('postgres://'):
-                uri = uri.replace('postgres://', 'postgresql://', 1)
+            if uri.startswith('postgres') and not uri.startswith('postgresql'):
+                uri = uri.replace('postgres', 'postgresql', 1)
+            if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
+                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql://', 1)
                 flavor = 'timescaledb'

meerschaum/connectors/sql/_create_engine.py

@@ -28,7 +28,7 @@ default_create_engine_args = {
 }
 flavor_configs = {
     'timescaledb'      : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -38,7 +38,7 @@ flavor_configs = {
         },
     },
     'postgresql'         : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -48,7 +48,7 @@ flavor_configs = {
         },
     },
     'citus'            : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -242,7 +242,7 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in ('timescaledb',) and self.flavor in _uri:
-            engine_str = engine_str.replace(f'{self.flavor}://', 'postgresql://')
+            engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
 
     if debug:
         dprint(

meerschaum/connectors/sql/_instance.py

@@ -155,7 +155,9 @@ def _drop_old_temporary_tables(
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()
-    if refresh or not last_check or (now_ts - last_check) > 60:
+    if not last_check:
+        self._stale_temporary_tables_check_timestamp = 0
+    if refresh or (now_ts - last_check) < 60:
         self._stale_temporary_tables_check_timestamp = now_ts
         return self._drop_temporary_tables(debug=debug)
 

meerschaum/connectors/sql/_pipes.py

@@ -752,7 +752,7 @@ def get_pipe_data(
         debug = debug,
         **kw
     )
-
+    
     if is_dask:
         index_col = pipe.columns.get('datetime', None)
         kw['index_col'] = index_col
@@ -1478,43 +1478,11 @@ def sync_pipe_inplace(
     from meerschaum.utils.misc import generate_password
     from meerschaum.utils.debug import dprint
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
-    metadef = self.get_pipe_metadef(
-        pipe,
-        params = params,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
-    )
-    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
-    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
-    internal_schema = self.internal_schema
-    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
-
-    if not pipe.exists(debug=debug):
-        create_pipe_query = get_create_table_query(
-            metadef,
-            pipe.target,
-            self.flavor,
-            schema = self.get_pipe_schema(pipe),
-        )
-        result = self.exec(create_pipe_query, debug=debug)
-        if result is None:
-            return False, f"Could not insert new data into {pipe} from its SQL query definition."
-        if not self.create_indices(pipe, debug=debug):
-            warn(f"Failed to create indices for {pipe}. Continuing...")
-
-        rowcount = pipe.get_rowcount(debug=debug)
-        return True, f"Inserted {rowcount}, updated 0 rows."
-
-    session = sqlalchemy_orm.Session(self.engine)
-    connectable = session if self.flavor != 'duckdb' else self
-
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
         return '-' + transact_id + '_' + label + '_' + pipe.target
 
+    internal_schema = self.internal_schema
     temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
     temp_tables = {
         table_root: get_temp_table_name(table_root)
@@ -1528,6 +1496,17 @@ def sync_pipe_inplace(
         )
         for table_root, table_name_raw in temp_tables.items()
     }
+    metadef = self.get_pipe_metadef(
+        pipe,
+        params = params,
+        begin = begin,
+        end = end,
+        check_existing = check_existing,
+        debug = debug,
+    )
+    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
+    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
 
     def clean_up_temp_tables(ready_to_drop: bool = False):
         log_success, log_msg = self._log_temporary_tables_creation(
@@ -1541,6 +1520,36 @@ def sync_pipe_inplace(
         )
         if not log_success:
             warn(log_msg)
+        drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(
+            refresh = False,
+            debug = debug,
+        )
+        if not drop_stale_success:
+            warn(drop_stale_msg)
+        return drop_stale_success, drop_stale_msg
+
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    if not pipe.exists(debug=debug):
+        create_pipe_query = get_create_table_query(
+            metadef,
+            pipe.target,
+            self.flavor,
+            schema = self.get_pipe_schema(pipe),
+        )
+        result = self.exec(create_pipe_query, debug=debug)
+        if result is None:
+            _ = clean_up_temp_tables()
+            return False, f"Could not insert new data into {pipe} from its SQL query definition."
+
+        if not self.create_indices(pipe, debug=debug):
+            warn(f"Failed to create indices for {pipe}. Continuing...")
+
+        rowcount = pipe.get_rowcount(debug=debug)
+        _ = clean_up_temp_tables()
+        return True, f"Inserted {rowcount}, updated 0 rows."
+
+    session = sqlalchemy_orm.Session(self.engine)
+    connectable = session if self.flavor != 'duckdb' else self
 
     create_new_query = get_create_table_query(
         metadef,
@@ -1908,10 +1917,6 @@ def sync_pipe_inplace(
     )
     _ = clean_up_temp_tables(ready_to_drop=True)
 
-    drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(refresh=False, debug=debug)
-    if not drop_stale_success:
-        warn(drop_stale_msg)
-
     return True, msg
 
 
@@ -2372,6 +2377,16 @@ def get_pipe_columns_types(
     """
     if not pipe.exists(debug=debug):
         return {}
+
+    if self.flavor == 'duckdb':
+        from meerschaum.utils.sql import get_table_cols_types
+        return get_table_cols_types(
+            pipe.target,
+            self,
+            flavor = self.flavor,
+            schema = self.schema,
+        )
+
     table_columns = {}
     try:
         pipe_table = self.get_pipe_table(pipe, debug=debug)

meerschaum/connectors/sql/_plugins.py

@@ -108,9 +108,7 @@ def get_plugin_version(
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
     sqlalchemy = attempt_import('sqlalchemy')
-
     query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
-
     return self.value(query, debug=debug)
 
 def get_plugin_user_id(

meerschaum/connectors/sql/_sql.py

@@ -943,17 +943,15 @@ def psql_insert_copy(
         ) for row in data_iter
     )
 
+    table_name = sql_item_name(table.name, 'postgresql', table.schema)
+    columns = ', '.join(f'"{k}"' for k in keys)
+    sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
-        s_buf = StringIO()
-        writer = csv.writer(s_buf)
-        writer.writerows(data_iter)
-        s_buf.seek(0)
-
-        columns = ', '.join(f'"{k}"' for k in keys)
-        table_name = sql_item_name(table.name, 'postgresql', table.schema)
-        sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
-        cur.copy_expert(sql=sql, file=s_buf)
+        with cur.copy(sql) as copy:
+            writer = csv.writer(copy)
+            writer.writerows(data_iter)
 
 
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':

meerschaum/core/User/_User.py

@@ -7,22 +7,157 @@ User class definition
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, Dict, Any
-
-pwd_context = None
-def get_pwd_context():
-    global pwd_context
-    if pwd_context is None:
-        from meerschaum.config.static import STATIC_CONFIG
-        from meerschaum.utils.packages import attempt_import
-        hash_config = STATIC_CONFIG['users']['password_hash']
-        passlib_context = attempt_import('passlib.context')
-        pwd_context = passlib_context.CryptContext(
-            schemes = hash_config['schemes'],
-            default = hash_config['default'],
-            pbkdf2_sha256__default_rounds = hash_config['pbkdf2_sha256__default_rounds']
+import os
+import hashlib
+import hmac
+from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
+from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn
+
+
+__all__ = ('hash_password', 'verify_password', 'User')
+
+def hash_password(
+        password: str,
+        salt: Optional[bytes] = None,
+        rounds: Optional[int] = None,
+    ) -> str:
+    """
+    Return an encoded hash string from the given password.
+
+    Parameters
+    ----------
+    password: str
+        The password to be hashed.
+
+    salt: Optional[str], default None
+        If provided, use these bytes for the salt in the hash.
+        Otherwise defaults to 16 random bytes.
+
+    rounds: Optional[int], default None
+        If provided, use this number of rounds to generate the hash.
+        Defaults to 3,000,000.
+        
+    Returns
+    -------
+    An encoded hash string to be stored in a database.
+    See the `passlib` documentation on the string format:
+    https://passlib.readthedocs.io/en/stable/lib/passlib.hash.pbkdf2_digest.html#format-algorithm
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    if password is None:
+        password = ''
+    if salt is None:
+        salt = os.urandom(hash_config['salt_bytes'])
+    if rounds is None:
+        rounds = hash_config['pbkdf2_sha256__default_rounds']
+
+    pw_hash = hashlib.pbkdf2_hmac(
+        hash_config['algorithm_name'],
+        password.encode('utf-8'),
+        salt,
+        rounds,
+    ) 
+    return (
+        f"$pbkdf2-{hash_config['algorithm_name']}"
+        + f"${hash_config['pbkdf2_sha256__default_rounds']}"
+        + '$' + ab64_encode(salt).decode('utf-8')
+        + '$' + ab64_encode(pw_hash).decode('utf-8')
+    )
+
+
+def verify_password(
+        password: str,
+        password_hash: str,
+    ) -> bool:
+    """
+    Return `True` if the password matches the provided hash.
+
+    Parameters
+    ----------
+    password: str
+        The password to be checked.
+
+    password_hash: str
+        The encoded hash string as generated from `hash_password()`.
+
+    Returns
+    -------
+    A `bool` indicating whether `password` matches `password_hash`.
+    """
+    if password is None or password_hash is None:
+        return False
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    try:
+        digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]
+        algorithm_name = digest.split('-')[-1]
+        salt = ab64_decode(encoded_salt)
+        checksum = ab64_decode(encoded_checksum)
+        rounds = int(rounds_str)
+    except Exception as e:
+        warn(f"Failed to extract context from password hash '{password_hash}'. Is it corrupted?")
+        return False
+
+    return hmac.compare_digest(
+        checksum,
+        hashlib.pbkdf2_hmac(
+            algorithm_name,
+            password.encode('utf-8'),
+            salt,
+            rounds,
         )
-    return pwd_context
+    )
+
+_BASE64_STRIP = b"=\n"
+_BASE64_PAD1 = b"="
+_BASE64_PAD2 = b"=="
+
+def ab64_encode(data):
+    return b64s_encode(data).replace(b"+", b".")
+
+def ab64_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses custom ``./`` altchars, but supports decoding normal ``+/`` altchars as well.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError:
+            raise ValueError("string argument should contain only ASCII characters")
+    return b64s_decode(data.replace(b".", b"+"))
+
+
+def b64s_encode(data):
+    return b2a_base64(data).rstrip(_BASE64_STRIP)
+
+def b64s_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses default ``+/`` altchars.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError as ue:
+            raise ValueError("string argument should contain only ASCII characters") from ue
+    off = len(data) & 3
+    if off == 0:
+        pass
+    elif off == 2:
+        data += _BASE64_PAD2
+    elif off == 3:
+        data += _BASE64_PAD1
+    else:  # off == 1
+        raise ValueError("Invalid base64 input")
+    try:
+        return a2b_base64(data)
+    except _BinAsciiError as err:
+        raise TypeError(err) from err
+
 
 class User:
     """
@@ -42,7 +177,6 @@ class User:
         if password is None:
             password = ''
         self.password = password
-        self.password_hash = get_pwd_context().hash(password)
         self.username = username
         self.email = email
         self.type = type
@@ -80,3 +214,11 @@ class User:
     @user_id.setter
     def user_id(self, user_id):
         self._user_id = user_id
+
+    @property
+    def password_hash(self):
+        _password_hash = self.__dict__.get('_password_hash', None)
+        if _password_hash is not None:
+            return _password_hash
+        self._password_hash = hash_password(self.password)
+        return self._password_hash

meerschaum/core/User/__init__.py

@@ -6,4 +6,4 @@
 Manager users' metadata via the User class
 """
 
-from meerschaum.core.User._User import User
+from meerschaum.core.User._User import User, hash_password, verify_password

meerschaum/plugins/_Plugin.py

@@ -209,7 +209,7 @@ class Plugin:
         def parse_gitignore() -> 'Set[str]':
             gitignore_path = pathlib.Path(path) / '.gitignore'
             if not gitignore_path.exists():
-                return set()
+                return set(default_patterns_to_ignore)
             with open(gitignore_path, 'r', encoding='utf-8') as f:
                 gitignore_text = f.read()
             return set(pathspec.PathSpec.from_lines(

meerschaum/plugins/__init__.py

@@ -247,6 +247,26 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 _warn(f"Unable to create lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")
 
     with _locks['internal_plugins']:
+
+        try:
+            from importlib.metadata import entry_points
+        except ImportError:
+            importlib_metadata = attempt_import('importlib_metadata', lazy=False)
+            entry_points = importlib_metadata.entry_points
+
+        ### NOTE: Allow plugins to be installed via `pip`.
+        packaged_plugin_paths = []
+        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        for ep in discovered_packaged_plugins_eps:
+            module_name = ep.name
+            for package_file_path in ep.dist.files:
+                if package_file_path.suffix != '.py':
+                    continue
+                if str(package_file_path) == f'{module_name}.py':
+                    packaged_plugin_paths.append(package_file_path.locate())
+                elif str(package_file_path) == f'{module_name}/__init__.py':
+                    packaged_plugin_paths.append(package_file_path.locate().parent)
+
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
@@ -255,7 +275,6 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
 
-
         existing_symlinked_paths = [
             (PLUGINS_RESOURCES_PATH / item) 
             for item in os.listdir(PLUGINS_RESOURCES_PATH)
@@ -275,6 +294,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 for plugins_path in PLUGINS_DIR_PATHS
             ]
         ))
+        plugins_to_be_symlinked.extend(packaged_plugin_paths)
 
         ### Check for duplicates.
         seen_plugins = defaultdict(lambda: 0)
@@ -538,6 +558,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
     ]
     plugins = tuple(plugin for plugin in _plugins if plugin.is_installed(try_import=try_import))
     if len(to_load) == 1:
+        if len(plugins) == 0:
+            raise ValueError(f"Plugin '{to_load[0]}' is not installed.")
         return plugins[0]
     return plugins
 

meerschaum/utils/daemon/Daemon.py

@@ -15,9 +15,11 @@ import signal
 import sys
 import time
 import traceback
+from functools import partial
 from datetime import datetime, timezone
 from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
 from meerschaum.config import get_config
+from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH
 from meerschaum.config._patch import apply_patch_to_config
 from meerschaum.utils.warnings import warn, error
@@ -139,11 +141,12 @@ class Daemon:
         Nothing — this will exit the parent process.
         """
         import platform, sys, os, traceback
-        from meerschaum.config._paths import LOGS_RESOURCES_PATH
+        from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
         from meerschaum.utils.warnings import warn
-        daemons_error_log_path = LOGS_RESOURCES_PATH / 'daemons_error.log'
-
+        from meerschaum.config import get_config
         daemon = attempt_import('daemon')
+        lines = get_config('jobs', 'terminal', 'lines')
+        columns = get_config('jobs','terminal', 'columns')
 
         if platform.system() == 'Windows':
             return False, "Windows is no longer supported."
@@ -164,10 +167,16 @@ class Daemon:
         )
 
         log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
-        self._log_refresh_timer = RepeatTimer(log_refresh_seconds, self.rotating_log.refresh_files)
+        self._log_refresh_timer = RepeatTimer(
+            log_refresh_seconds,
+            partial(self.rotating_log.refresh_files, start_interception=True),
+        )
 
         try:
+            os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
             with self._daemon_context:
+                os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
+                self.rotating_log.refresh_files(start_interception=True)
                 try:
                     with open(self.pid_path, 'w+', encoding='utf-8') as f:
                         f.write(str(os.getpid()))
@@ -192,11 +201,11 @@ class Daemon:
                 return result
         except Exception as e:
             daemon_error = traceback.format_exc()
-            with open(LOGS_RESOURCES_PATH, 'a+', encoding='utf-8') as f:
+            with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
                 f.write(daemon_error)
 
         if daemon_error:
-            warn("Encountered an error while starting the daemon '{self}':\n{daemon_error}")
+            warn(f"Encountered an error while starting the daemon '{self}':\n{daemon_error}")
 
 
     def _capture_process_timestamp(
@@ -456,6 +465,9 @@ class Daemon:
         Handle `SIGINT` within the Daemon context.
         This method is injected into the `DaemonContext`.
         """
+        #  from meerschaum.utils.daemon.FileDescriptorInterceptor import STOP_READING_FD_EVENT
+        #  STOP_READING_FD_EVENT.set()
+        self.rotating_log.stop_log_fd_interception(unused_only=False)
         timer = self.__dict__.get('_log_refresh_timer', None)
         if timer is not None:
             timer.cancel()
@@ -464,10 +476,18 @@ class Daemon:
         if daemon_context is not None:
             daemon_context.close()
 
-        self.rotating_log.stop_log_fd_interception()
-
         _close_pools()
-        raise SystemExit(0)
+        import threading
+        for thread in threading.enumerate():
+            if thread.name == 'MainThread':
+                continue
+            try:
+                if thread.is_alive():
+                    stack = traceback.format_stack(sys._current_frames()[thread.ident])
+                    thread.join()
+            except Exception as e:
+                warn(traceback.format_exc())
+        raise KeyboardInterrupt()
 
 
     def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
@@ -483,10 +503,8 @@ class Daemon:
         if daemon_context is not None:
             daemon_context.close()
 
-        self.rotating_log.stop_log_fd_interception()
-
         _close_pools()
-        raise SystemExit(1)
+        raise SystemExit(0)
 
  
     def _send_signal(
@@ -667,6 +685,7 @@ class Daemon:
             self.log_path,
             redirect_streams = True,
             write_timestamps = True,
+            timestamp_format = get_config('jobs', 'logs', 'timestamp_format'),
         )
         return self._rotating_log
 
@@ -905,6 +924,13 @@ class Daemon:
                 return False, msg
         if not keep_logs:
             self.rotating_log.delete()
+            try:
+                if self.log_offset_path.exists():
+                    self.log_offset_path.unlink()
+            except Exception as e:
+                msg = f"Failed to remove offset file for '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
         return True, "Success"