meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -26,8 +26,6 @@ from meerschaum.utils.typing import (
     SuccessTuple,
     Dict,
     List,
-    Iterable,
-    Generator,
 )
 from meerschaum.utils.warnings import warn, error
 
@@ -266,7 +264,6 @@ def sync(
                             **kw
                         )
                     )
-
             except Exception as e:
                 get_console().print_exception(
                     suppress=[
@@ -369,6 +366,11 @@ def sync(
 
         ### Cast to a dataframe and ensure datatypes are what we expect.
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+
+        ### Capture `numeric` and `json` columns.
+        self._persist_new_json_columns(df, debug=debug)
+        self._persist_new_numeric_columns(df, debug=debug)
+
         if debug:
             dprint(
                 "DataFrame to sync:\n"
@@ -554,14 +556,15 @@ def exists(
 
 
 def filter_existing(
-        self,
-        df: 'pd.DataFrame',
-        safe_copy: bool = True,
-        date_bound_only: bool = False,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw
-    ) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
+    self,
+    df: 'pd.DataFrame',
+    safe_copy: bool = True,
+    date_bound_only: bool = False,
+    include_unchanged_columns: bool = False,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw
+) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
     """
     Inspect a dataframe and filter out rows which already exist in the pipe.
 
@@ -569,7 +572,7 @@ def filter_existing(
     ----------
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
-        
+
     safe_copy: bool, default True
         If `True`, create a copy before comparing and modifying the dataframes.
         Setting to `False` may mutate the DataFrames.
@@ -578,6 +581,10 @@ def filter_existing(
     date_bound_only: bool, default False
         If `True`, only use the datetime index to fetch the sample dataframe.
 
+    include_unchanged_columns: bool, default False
+        If `True`, include the backtrack columns which haven't changed in the update dataframe.
+        This is useful if you can't update individual keys.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -605,7 +612,7 @@ def filter_existing(
     from meerschaum.config import get_config
     pd = import_pandas()
     pandas = attempt_import('pandas')
-    if not 'dataframe' in str(type(df)).lower():
+    if 'dataframe' not in str(type(df)).lower():
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
     is_dask = 'dask' in df.__module__
     if is_dask:
@@ -615,8 +622,21 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
+
+    def get_empty_df():
+        empty_df = pd.DataFrame([])
+        dtypes = dict(df.dtypes) if df is not None else {}
+        dtypes.update(self.dtypes)
+        pd_dtypes = {
+            col: to_pandas_dtype(str(typ))
+            for col, typ in dtypes.items()
+        }
+        return add_missing_cols_to_df(empty_df, pd_dtypes)
+
     if df is None:
-        return df, df, df
+        empty_df = get_empty_df()
+        return empty_df, empty_df, empty_df
+
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -633,7 +653,7 @@ def filter_existing(
             if min_dt_val is not None and 'datetime' in str(dt_type)
             else min_dt_val
         )
-    except Exception as e:
+    except Exception:
         min_dt = None
     if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(min_dt)).lower():
@@ -643,7 +663,7 @@ def filter_existing(
         begin = (
             round_time(
                 min_dt,
-                to = 'down'
+                to='down'
             ) - timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
@@ -661,7 +681,7 @@ def filter_existing(
             if max_dt_val is not None and 'datetime' in str(dt_type)
             else max_dt_val
         )
-    except Exception as e:
+    except Exception:
         import traceback
         traceback.print_exc()
         max_dt = None
@@ -674,14 +694,14 @@ def filter_existing(
         end = (
             round_time(
                 max_dt,
-                to = 'down'
+                to='down'
             ) + timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
         end = max_dt + 1
 
     if max_dt is not None and min_dt is not None and min_dt > max_dt:
-        warn(f"Detected minimum datetime greater than maximum datetime.")
+        warn("Detected minimum datetime greater than maximum datetime.")
 
     if begin is not None and end is not None and begin > end:
         if isinstance(begin, datetime):
@@ -710,13 +730,18 @@ def filter_existing(
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
 
     backtrack_df = self.get_data(
-        begin = begin,
-        end = end,
-        chunksize = chunksize,
-        params = params,
-        debug = debug,
+        begin=begin,
+        end=end,
+        chunksize=chunksize,
+        params=params,
+        debug=debug,
         **kw
     )
+    if backtrack_df is None:
+        if debug:
+            dprint(f"No backtrack data was found for {self}.")
+        return df, get_empty_df(), df
+
     if debug:
         dprint(f"Existing data for {self}:\n" + str(backtrack_df), **kw)
         dprint(f"Existing dtypes for {self}:\n" + str(backtrack_df.dtypes))
@@ -743,18 +768,19 @@ def filter_existing(
         filter_unseen_df(
             backtrack_df,
             df,
-            dtypes = {
+            dtypes={
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
-            safe_copy = safe_copy,
-            debug = debug
+            safe_copy=safe_copy,
+            debug=debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
     serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+
     def deserializer(x):
         return json.loads(x) if isinstance(x, str) else x
 
@@ -767,12 +793,12 @@ def filter_existing(
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
-        delta_df.fillna(NA),
-        backtrack_df.fillna(NA),
-        how = 'left',
-        on = on_cols,
-        indicator = True,
-        suffixes = ('', '_old'),
+        delta_df.infer_objects(copy=False).fillna(NA),
+        backtrack_df.infer_objects(copy=False).fillna(NA),
+        how='left',
+        on=on_cols,
+        indicator=True,
+        suffixes=('', '_old'),
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
@@ -782,20 +808,13 @@ def filter_existing(
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None
-    cols = list(backtrack_df.columns)
+    cols = list(delta_df.columns)
 
     unseen_df = (
-        (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        ) if not is_dask else (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        )
+        joined_df
+        .where(new_rows_mask)
+        .dropna(how='all')[cols]
+        .reset_index(drop=True)
     ) if on_cols else delta_df
 
     ### Rows that have already been inserted but values have changed.
@@ -804,20 +823,33 @@ def filter_existing(
         .where(~new_rows_mask)
         .dropna(how='all')[cols]
         .reset_index(drop=True)
-    ) if on_cols else None
+    ) if on_cols else get_empty_df()
+
+    if include_unchanged_columns and on_cols:
+        unchanged_backtrack_cols = [
+            col
+            for col in backtrack_df.columns
+            if col in on_cols or col not in update_df.columns
+        ]
+        update_df = merge(
+            backtrack_df[unchanged_backtrack_cols],
+            update_df,
+            how='inner',
+            on=on_cols,
+        )
 
     return unseen_df, update_df, delta_df
 
 
 @staticmethod
 def _get_chunk_label(
-        chunk: Union[
-            'pd.DataFrame',
-            List[Dict[str, Any]],
-            Dict[str, List[Any]]
-        ],
-        dt_col: str,
-    ) -> str:
+    chunk: Union[
+        'pd.DataFrame',
+        List[Dict[str, Any]],
+        Dict[str, List[Any]]
+    ],
+    dt_col: str,
+) -> str:
     """
     Return the min - max label for the chunk.
     """
@@ -870,3 +902,52 @@ def get_num_workers(self, workers: Optional[int] = None) -> int:
         (desired_workers - current_num_connections),
         1,
     )
+
+
+def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new numeric columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_numeric_cols
+    numeric_cols = get_numeric_cols(df)
+    existing_numeric_cols = [col for col, typ in self.dtypes.items() if typ == 'numeric']
+    new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
+    if not new_numeric_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'numeric' for col in numeric_cols})
+    self.parameters['dtypes'] = dtypes
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update NUMERIC dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"
+
+
+def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new JSON columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_json_cols
+    json_cols = get_json_cols(df)
+    existing_json_cols = [col for col, typ in self.dtypes.items() if typ == 'json']
+    new_json_cols = [col for col in json_cols if col not in existing_json_cols]
+    if not new_json_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'json' for col in json_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update JSON dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"

meerschaum/core/Pipe/_verify.py

@@ -12,17 +12,17 @@ from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
 def verify(
-        self,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        bounded: Optional[bool] = None,
-        deduplicate: bool = False,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    self,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    bounded: Optional[bool] = None,
+    deduplicate: bool = False,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify the contents of the pipe by resyncing its interval.
 

meerschaum/core/User/_User.py

@@ -11,7 +11,7 @@ import os
 import hashlib
 import hmac
 from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
-from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn
 
@@ -19,10 +19,10 @@ from meerschaum.utils.warnings import warn
 __all__ = ('hash_password', 'verify_password', 'User')
 
 def hash_password(
-        password: str,
-        salt: Optional[bytes] = None,
-        rounds: Optional[int] = None,
-    ) -> str:
+    password: str,
+    salt: Optional[bytes] = None,
+    rounds: Optional[int] = None,
+) -> str:
     """
     Return an encoded hash string from the given password.
 
@@ -68,9 +68,9 @@ def hash_password(
 
 
 def verify_password(
-        password: str,
-        password_hash: str,
-    ) -> bool:
+    password: str,
+    password_hash: str,
+) -> bool:
     """
     Return `True` if the password matches the provided hash.
 
@@ -197,26 +197,28 @@ class User:
         return self._attributes
 
     @property
-    def instance_connector(self) -> meerschaum.connectors.Connector:
-        """ """
+    def instance_connector(self) -> 'mrsm.connectors.Connector':
         from meerschaum.connectors.parse import parse_instance_keys
         if '_instance_connector' not in self.__dict__:
             self._instance_connector = parse_instance_keys(self._instance_keys)
         return self._instance_connector
 
     @property
-    def user_id(self) -> int:
+    def user_id(self) -> Union[int, str, None]:
         """NOTE: This causes recursion with the API,
               so don't try to get fancy with read-only attributes.
         """
         return self._user_id
 
     @user_id.setter
-    def user_id(self, user_id):
+    def user_id(self, user_id: Union[int, str, None]):
         self._user_id = user_id
 
     @property
     def password_hash(self):
+        """
+        Return the hash of the user's password.
+        """
         _password_hash = self.__dict__.get('_password_hash', None)
         if _password_hash is not None:
             return _password_hash

meerschaum/jobs/_Job.py

@@ -10,10 +10,7 @@ from __future__ import annotations
 
 import shlex
 import asyncio
-import threading
-import json
 import pathlib
-import os
 import sys
 import traceback
 from functools import partial
@@ -60,9 +57,10 @@ class Job:
         sysargs: Union[List[str], str, None] = None,
         env: Optional[Dict[str, str]] = None,
         executor_keys: Optional[str] = None,
+        delete_after_completion: bool = False,
         _properties: Optional[Dict[str, Any]] = None,
-        _rotating_log = None,
-        _stdin_file = None,
+        _rotating_log=None,
+        _stdin_file=None,
         _status_hook: Optional[Callable[[], str]] = None,
         _result_hook: Optional[Callable[[], SuccessTuple]] = None,
         _externally_managed: bool = False,
@@ -85,6 +83,9 @@ class Job:
         executor_keys: Optional[str], default None
             If provided, execute the job remotely on an API instance, e.g. 'api:main'.
 
+        delete_after_completion: bool, default False
+            If `True`, delete this job when it has finished executing.
+
         _properties: Optional[Dict[str, Any]], default None
             If provided, use this to patch the daemon's properties.
         """
@@ -146,6 +147,9 @@ class Job:
         if env:
             self._properties_patch.update({'env': env})
 
+        if delete_after_completion:
+            self._properties_patch.update({'delete_after_completion': delete_after_completion})
+
         daemon_sysargs = (
             self._daemon.properties.get('target', {}).get('args', [None])[0]
             if self._daemon is not None
@@ -199,13 +203,11 @@ class Job:
             jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
             daemon_dir = jobs_dir / daemon_id
             pid_file = daemon_dir / 'process.pid'
-            properties_path = daemon_dir / 'properties.json'
-            pickle_path = daemon_dir / 'pickle.pkl'
 
             if pid_file.exists():
                 with open(pid_file, 'r', encoding='utf-8') as f:
                     daemon_pid = int(f.read())
-                
+
                 if pid != daemon_pid:
                     raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
             else:
@@ -245,7 +247,7 @@ class Job:
             return True, f"{self} is already running."
 
         success, msg = self.daemon.run(
-            keep_daemon_output=True,
+            keep_daemon_output=(not self.delete_after_completion),
             allow_dirty_run=True,
         )
         if not success:
@@ -407,7 +409,6 @@ class Job:
         )
         return asyncio.run(monitor_logs_coroutine)
 
-
     async def monitor_logs_async(
         self,
         callback_function: Callable[[str], None] = partial(print, end='', flush=True),
@@ -418,8 +419,8 @@ class Job:
         strip_timestamps: bool = False,
         accept_input: bool = True,
         _logs_path: Optional[pathlib.Path] = None,
-        _log = None,
-        _stdin_file = None,
+        _log=None,
+        _stdin_file=None,
         debug: bool = False,
     ):
         """
@@ -466,6 +467,7 @@ class Job:
                 input_callback_function=input_callback_function,
                 stop_callback_function=stop_callback_function,
                 stop_on_exit=stop_on_exit,
+                strip_timestamps=strip_timestamps,
                 accept_input=accept_input,
                 debug=debug,
             )
@@ -557,7 +559,6 @@ class Job:
                 for task in pending:
                     task.cancel()
             except asyncio.exceptions.CancelledError:
-                print('cancelled?')
                 pass
             finally:
                 combined_event.set()
@@ -870,7 +871,9 @@ class Job:
         """
         Return the job's Daemon label (joined sysargs).
         """
-        return shlex.join(self.sysargs).replace(' + ', '\n+ ')
+        from meerschaum._internal.arguments import compress_pipeline_sysargs
+        sysargs = compress_pipeline_sysargs(self.sysargs)
+        return shlex.join(sysargs).replace(' + ', '\n+ ')
 
     @property
     def _externally_managed_file(self) -> pathlib.Path:
@@ -916,6 +919,16 @@ class Job:
         self._env = {**default_env, **_env}
         return self._env
 
+    @property
+    def delete_after_completion(self) -> bool:
+        """
+        Return whether this job is configured to delete itself after completion.
+        """
+        if '_delete_after_completion' in self.__dict__:
+            return self.__dict__.get('_delete_after_completion', False)
+
+        self._delete_after_completion = self.daemon.properties.get('delete_after_completion', False)
+        return self._delete_after_completion
 
     def __str__(self) -> str:
         sysargs = self.sysargs

meerschaum/jobs/__init__.py

@@ -9,9 +9,9 @@ Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
 import pathlib
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, Optional, List, Callable, Any, SuccessTuple
+from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
-from meerschaum.jobs._Job import Job, StopMonitoringLogs
+from meerschaum.jobs._Job import Job
 from meerschaum.jobs._Executor import Executor
 
 __all__ = (
@@ -403,9 +403,14 @@ def get_executor_keys_from_context() -> str:
     if _context_keys is not None:
         return _context_keys
 
+    from meerschaum.config import get_config
     from meerschaum.config.paths import ROOT_DIR_PATH, DEFAULT_ROOT_DIR_PATH
     from meerschaum.utils.misc import is_systemd_available
 
+    configured_executor = get_config('meerschaum', 'executor', warn=False)
+    if configured_executor is not None:
+        return configured_executor
+
     _context_keys = (
         'systemd'
         if is_systemd_available() and ROOT_DIR_PATH == DEFAULT_ROOT_DIR_PATH

meerschaum/jobs/systemd.py

@@ -42,7 +42,12 @@ class SystemdExecutor(Executor):
         return [
             service_name[len('mrsm-'):(-1 * len('.service'))]
             for service_name in os.listdir(SYSTEMD_USER_RESOURCES_PATH)
-            if service_name.startswith('mrsm-') and service_name.endswith('.service')
+            if (
+                service_name.startswith('mrsm-')
+                and service_name.endswith('.service')
+                ### Check for broken symlinks.
+                and (SYSTEMD_USER_RESOURCES_PATH / service_name).exists()
+            )
         ]
 
     def get_job_exists(self, name: str, debug: bool = False) -> bool:
@@ -146,6 +151,11 @@ class SystemdExecutor(Executor):
             STATIC_CONFIG['environment']['systemd_log_path']: service_logs_path.as_posix(),
             STATIC_CONFIG['environment']['systemd_result_path']: result_path.as_posix(),
             STATIC_CONFIG['environment']['systemd_stdin_path']: socket_path.as_posix(),
+            STATIC_CONFIG['environment']['systemd_delete_job']: (
+                '1'
+                if job.delete_after_completion
+                else '0',
+            ),
         })
 
         ### Allow for user-defined environment variables.
@@ -603,7 +613,8 @@ class SystemdExecutor(Executor):
 
         check_timeout_interval = get_config('jobs', 'check_timeout_interval_seconds')
         loop_start = time.perf_counter()
-        while (time.perf_counter() - loop_start) < get_config('jobs', 'timeout_seconds'):
+        timeout_seconds = get_config('jobs', 'timeout_seconds')
+        while (time.perf_counter() - loop_start) < timeout_seconds:
             if self.get_job_status(name, debug=debug) == 'stopped':
                 return True, 'Success'
 
@@ -630,12 +641,14 @@ class SystemdExecutor(Executor):
         Delete a job's service.
         """
         from meerschaum.config.paths import SYSTEMD_LOGS_RESOURCES_PATH
+        job = self.get_hidden_job(name, debug=debug)
 
-        _ = self.stop_job(name, debug=debug)
-        _ = self.run_command(
-            ['disable', self.get_service_name(name, debug=debug)],
-            debug=debug,
-        )
+        if not job.delete_after_completion:
+            _ = self.stop_job(name, debug=debug)
+            _ = self.run_command(
+                ['disable', self.get_service_name(name, debug=debug)],
+                debug=debug,
+            )
 
         service_job_path = self.get_service_job_path(name, debug=debug)
         try:
@@ -666,7 +679,6 @@ class SystemdExecutor(Executor):
                     warn(e)
                     return False, str(e)
 
-        job = self.get_hidden_job(name, debug=debug)
         _ = job.delete()
 
         return self.run_command(['daemon-reload'], debug=debug)