meerschaum 2.3.0rc3__py3-none-any.whl → 2.3.1__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -11,14 +11,15 @@ import time
 import json
 from io import StringIO
 from datetime import datetime
+
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
 
 def pipe_r_url(
-        pipe: mrsm.Pipe
-    ) -> str:
+    pipe: mrsm.Pipe
+) -> str:
     """Return a relative URL path from a Pipe's keys."""
     from meerschaum.config.static import STATIC_CONFIG
     location_key = pipe.location_key
@@ -30,10 +31,10 @@ def pipe_r_url(
     )
 
 def register_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """Submit a POST to the API to register a new Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -59,11 +60,11 @@ def register_pipe(
 
 
 def edit_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        patch: bool = False,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    patch: bool = False,
+    debug: bool = False,
+) -> SuccessTuple:
     """Submit a PATCH to the API to edit an existing Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -89,14 +90,14 @@ def edit_pipe(
 
 
 def fetch_pipes_keys(
-        self,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[str]] = None,
-        tags: Optional[List[str]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False
-    ) -> Union[List[Tuple[str, str, Union[str, None]]]]:
+    self,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[str]] = None,
+    tags: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False
+) -> Union[List[Tuple[str, str, Union[str, None]]]]:
     """
     Fetch registered Pipes' keys from the API.
     
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
 
 
 def sync_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
@@ -303,10 +304,10 @@ def sync_pipe(
 
 
 def delete_pipe(
-        self,
-        pipe: Optional[meerschaum.Pipe] = None,
-        debug: bool = None,        
-    ) -> SuccessTuple:
+    self,
+    pipe: Optional[meerschaum.Pipe] = None,
+    debug: bool = None,        
+) -> SuccessTuple:
     """Delete a Pipe and drop its table."""
     if pipe is None:
         error(f"Pipe cannot be None.")
@@ -327,17 +328,17 @@ def delete_pipe(
 
 
 def get_pipe_data(
-        self,
-        pipe: meerschaum.Pipe,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[str, datetime, int, None] = None,
-        end: Union[str, datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        as_chunks: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[pandas.DataFrame, None]:
+    self,
+    pipe: meerschaum.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    as_chunks: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
     r_url = pipe_r_url(pipe)
     chunks_list = []
@@ -389,10 +390,10 @@ def get_pipe_data(
 
 
 def get_pipe_id(
-        self,
-        pipe: meerschuam.Pipe,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: meerschuam.Pipe,
+    debug: bool = False,
+) -> int:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
@@ -411,10 +412,10 @@ def get_pipe_id(
 
 
 def get_pipe_attributes(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, Any]:
+    self,
+    pipe: meerschaum.Pipe,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """Get a Pipe's attributes from the API
 
     Parameters
@@ -437,12 +438,12 @@ def get_pipe_attributes(
 
 
 def get_sync_time(
-        self,
-        pipe: 'meerschaum.Pipe',
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        debug: bool = False,
-    ) -> Union[datetime, int, None]:
+    self,
+    pipe: 'meerschaum.Pipe',
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    debug: bool = False,
+) -> Union[datetime, int, None]:
     """Get a Pipe's most recent datetime value from the API.
 
     Parameters
@@ -492,10 +493,10 @@ def get_sync_time(
 
 
 def pipe_exists(
-        self,
-        pipe: 'meerschaum.Pipe',
-        debug: bool = False
-    ) -> bool:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> bool:
     """Check the API to see if a Pipe exists.
 
     Parameters
@@ -523,9 +524,9 @@ def pipe_exists(
 
 
 def create_metadata(
-        self,
-        debug: bool = False
-    ) -> bool:
+    self,
+    debug: bool = False
+) -> bool:
     """Create metadata tables.
 
     Returns
@@ -547,14 +548,14 @@ def create_metadata(
 
 
 def get_pipe_rowcount(
-        self,
-        pipe: 'meerschaum.Pipe',
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        remote: bool = False,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    remote: bool = False,
+    debug: bool = False,
+) -> int:
     """Get a pipe's row count from the API.
 
     Parameters
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
 
 
 def drop_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """
     Drop a pipe's table but maintain its registration.
 
@@ -644,11 +645,11 @@ def drop_pipe(
 
 
 def clear_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete rows in a pipe's table.
 
@@ -666,7 +667,7 @@ def clear_pipe(
     kw.pop('location_keys', None)
     kw.pop('action', None)
     kw.pop('force', None)
-    return self.do_action(
+    return self.do_action_legacy(
         ['clear', 'pipes'],
         connector_keys = pipe.connector_keys,
         metric_keys = pipe.metric_key,
@@ -678,10 +679,10 @@ def clear_pipe(
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Union[Dict[str, str], None]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
     """
     Fetch the columns and types of the pipe's table.
 

meerschaum/jobs/_Job.py

@@ -62,8 +62,9 @@ class Job:
         _properties: Optional[Dict[str, Any]] = None,
         _rotating_log = None,
         _stdin_file = None,
-        _status_hook = None,
-        _result_hook = None,
+        _status_hook: Optional[Callable[[], str]] = None,
+        _result_hook: Optional[Callable[[], SuccessTuple]] = None,
+        _externally_managed: bool = False,
     ):
         """
         Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
@@ -133,7 +134,10 @@ class Job:
         if _result_hook is not None:
             self._result_hook = _result_hook
 
+        self._externally_managed = _externally_managed
         self._properties_patch = _properties or {}
+        if _externally_managed:
+            self._properties_patch.update({'externally_managed': _externally_managed})
 
         daemon_sysargs = (
             self._daemon.properties.get('target', {}).get('args', [None])[0]
@@ -154,9 +158,6 @@ class Job:
                 self._properties_patch.update({'restart': True})
                 break
 
-        if '--systemd' in self._sysargs:
-            self._properties_patch.update({'systemd': True})
-
     @staticmethod
     def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
         """
@@ -824,7 +825,11 @@ class Job:
         """
         Return a bool indicating whether this job should be displayed.
         """
-        return self.name.startswith('_') or self.name.startswith('.')
+        return (
+            self.name.startswith('_')
+            or self.name.startswith('.')
+            or self._is_externally_managed
+        )
 
     def check_restart(self) -> SuccessTuple:
         """
@@ -850,6 +855,33 @@ class Job:
         """
         return shlex.join(self.sysargs).replace(' + ', '\n+ ')
 
+    @property
+    def _externally_managed_file(self) -> pathlib.Path:
+        """
+        Return the path to the externally managed file.
+        """
+        return self.daemon.path / '.externally-managed'
+
+    def _set_externally_managed(self):
+        """
+        Set this job as externally managed.
+        """
+        self._externally_managed = True
+        try:
+            self._externally_managed_file.parent.mkdir(exist_ok=True, parents=True)
+            self._externally_managed_file.touch()
+        except Exception as e:
+            warn(e)
+
+    @property
+    def _is_externally_managed(self) -> bool:
+        """
+        Return whether this job is externally managed.
+        """
+        return self.executor_keys in (None, 'local') and (
+            self._externally_managed or self._externally_managed_file.exists()
+        )
+
     def __str__(self) -> str:
         sysargs = self.sysargs
         sysargs_str = shlex.join(sysargs) if sysargs else ''

meerschaum/jobs/__init__.py

@@ -16,6 +16,7 @@ from meerschaum.jobs._Executor import Executor
 
 __all__ = (
     'Job',
+    'systemd',
     'get_jobs',
     'get_filtered_jobs',
     'get_restart_jobs',
@@ -23,8 +24,8 @@ __all__ = (
     'get_stopped_jobs',
     'get_paused_jobs',
     'get_restart_jobs',
-    'Executor',
     'make_executor',
+    'Executor',
     'check_restart_jobs',
     'start_check_jobs_thread',
     'stop_check_jobs_thread',
@@ -72,7 +73,8 @@ def get_jobs(
         return {
             name: job
             for name, job in jobs.items()
-            if include_hidden or not job.hidden
+            if (include_hidden or not job.hidden) and not job._is_externally_managed
+
         }
 
     def _get_systemd_jobs():
@@ -387,7 +389,7 @@ def _install_healthcheck_job() -> SuccessTuple:
         return False, "Not running systemd."
 
     job = Job(
-        '.local_healthcheck',
+        '.local-healthcheck',
         ['restart', 'jobs', '-e', 'local', '--loop'],
         executor_keys='systemd',
     )

meerschaum/jobs/{_SystemdExecutor.py → systemd.py}

@@ -13,6 +13,7 @@ import asyncio
 import json
 import time
 import traceback
+import shutil
 from datetime import datetime, timezone
 from functools import partial
 
@@ -37,10 +38,10 @@ class SystemdExecutor(Executor):
         """
         Return a list of existing jobs, including hidden ones.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
+        from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
         return [
             service_name[len('mrsm-'):(-1 * len('.service'))]
-            for service_name in os.listdir(SYSTEMD_ROOT_RESOURCES_PATH)
+            for service_name in os.listdir(SYSTEMD_USER_RESOURCES_PATH)
             if service_name.startswith('mrsm-') and service_name.endswith('.service')
         ]
 
@@ -73,6 +74,13 @@ class SystemdExecutor(Executor):
         """
         return f"mrsm-{name.replace(' ', '-')}.service"
 
+    def get_service_job_path(self, name: str, debug: bool = False) -> pathlib.Path:
+        """
+        Return the path for the job's files under the root directory.
+        """
+        from meerschaum.config.paths import SYSTEMD_JOBS_RESOURCES_PATH
+        return SYSTEMD_JOBS_RESOURCES_PATH / name
+
     def get_service_symlink_file_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to where to create the service symlink.
@@ -84,8 +92,10 @@ class SystemdExecutor(Executor):
         """
         Return the path to a Job's service file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / self.get_service_name(name, debug=debug)
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / self.get_service_name(name, debug=debug)
+        )
 
     def get_service_logs_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
@@ -94,29 +104,22 @@ class SystemdExecutor(Executor):
         from meerschaum.config.paths import SYSTEMD_LOGS_RESOURCES_PATH
         return SYSTEMD_LOGS_RESOURCES_PATH / (self.get_service_name(name, debug=debug) + '.log')
 
-    def get_service_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
-        """
-        Return the path to the unit file for the socket (not the socket itself).
-        """
-        from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
-        return SYSTEMD_USER_RESOURCES_PATH / (
-            self.get_service_name(name, debug=debug).replace('.service', '.socket')
-        )
-
     def get_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to the FIFO file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / (self.get_service_name(name, debug=debug) + '.stdin')
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / (self.get_service_name(name, debug=debug) + '.stdin')
+        )
 
     def get_result_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to the result file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / (
-            self.get_service_name(name, debug=debug) + '.result.json'
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / (self.get_service_name(name, debug=debug) + '.result.json')
         )
 
     def get_service_file_text(self, name: str, sysargs: List[str], debug: bool = False) -> str:
@@ -191,17 +194,19 @@ class SystemdExecutor(Executor):
         """
         Return the hidden "sister" job to store a job's parameters.
         """
-        hidden_name = f'.systemd-{self.get_service_name(name, debug=debug)}' 
-
-        return Job(
-            hidden_name,
+        job = Job(
+            name,
             sysargs,
             executor_keys='local',
             _rotating_log=self.get_job_rotating_file(name, debug=debug),
             _stdin_file=self.get_job_stdin_file(name, debug=debug),
             _status_hook=partial(self.get_job_status, name),
             _result_hook=partial(self.get_job_result, name),
+            _externally_managed=True,
         )
+        job._set_externally_managed()
+        return job
+
 
     def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
         """
@@ -489,6 +494,7 @@ class SystemdExecutor(Executor):
 
         if name not in self._stdin_files:
             socket_path = self.get_socket_path(name, debug=debug)
+            socket_path.parent.mkdir(parents=True, exist_ok=True)
             self._stdin_files[name] = StdinFile(socket_path)
 
         return self._stdin_files[name]
@@ -504,6 +510,9 @@ class SystemdExecutor(Executor):
         socket_stdin = self.get_job_stdin_file(name, debug=debug)
         _ = socket_stdin.file_handler
 
+        ### Init the `externally_managed file`.
+        _ = self.get_hidden_job(name, debug=debug)
+
         with open(service_file_path, 'w+', encoding='utf-8') as f:
             f.write(self.get_service_file_text(name, sysargs, debug=debug))
 
@@ -594,6 +603,14 @@ class SystemdExecutor(Executor):
             debug=debug,
         )
 
+        service_job_path = self.get_service_job_path(name, debug=debug)
+        try:
+            if service_job_path.exists():
+                shutil.rmtree(service_job_path)
+        except Exception as e:
+            warn(e)
+            return False, str(e)
+
         service_logs_path = self.get_service_logs_path(name, debug=debug)
         logs_paths = [
             (SYSTEMD_LOGS_RESOURCES_PATH / name)

meerschaum/plugins/_Plugin.py

@@ -654,7 +654,7 @@ class Plugin:
             import ast, re
             ### NOTE: This technically would break 
             ### if `required` was the very first line of the file.
-            req_start_match = re.search(r'required(:\s*)?.*=', text)
+            req_start_match = re.search(r'\nrequired(:\s*)?.*=', text)
             if not req_start_match:
                 return []
             req_start = req_start_match.start()

meerschaum/plugins/__init__.py

@@ -28,7 +28,8 @@ _locks = {
     'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
 }
 __all__ = (
-    "Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
+    "Plugin", "make_action", "api_plugin", "dash_plugin", "web_page",
+    "import_plugins", "from_plugin_import",
     "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
     "pre_sync_hook", "post_sync_hook",
 )

meerschaum/utils/daemon/StdinFile.py

@@ -50,6 +50,7 @@ class StdinFile(io.TextIOBase):
         if self.file_path.exists():
             self.file_path.unlink()
 
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
         os.mkfifo(self.file_path.as_posix(), mode=0o600)
 
         self._fd = os.open(self.file_path, os.O_RDONLY | os.O_NONBLOCK)

meerschaum/utils/daemon/_names.py

@@ -18,7 +18,8 @@ _bank: Dict[str, Dict[str, List[str]]] = {
             'bright', 'dark', 'neon',
         ],
         'sizes': [
-            'big', 'small', 'large', 'huge', 'tiny', 'long', 'short', 'grand', 'mini', 'micro'
+            'big', 'small', 'large', 'huge', 'tiny', 'long', 'short', 'average', 'mini', 'micro',
+            'maximum', 'minimum', 'median',
         ],
         'personalities': [
             'groovy', 'cool', 'awesome', 'nice', 'fantastic', 'sweet', 'great', 'amazing',
@@ -26,29 +27,36 @@ _bank: Dict[str, Dict[str, List[str]]] = {
         ],
         'emotions': [
             'angry', 'happy', 'excited', 'suspicious', 'sad', 'thankful', 'grateful', 'satisfied',
+            'peaceful', 'ferocious', 'content',
         ],
         'sensations': [
             'sleepy', 'awake', 'alert', 'thirsty', 'comfy', 'warm', 'cold', 'chilly', 'soft',
-            'smooth', 'chunky',
+            'smooth', 'chunky', 'hungry',
         ],
         'materials': [
             'golden', 'silver', 'metal', 'plastic', 'wool', 'wooden', 'nylon', 'fuzzy', 'silky',
+            'suede', 'vinyl',
         ],
         'qualities': [
             'expensive', 'cheap', 'premier', 'best', 'favorite', 'better', 'good', 'affordable',
+            'organic', 'electric',
         ],
     },
     'nouns' : {
         'animals': [
-            'mouse', 'fox', 'horse', 'dragon', 'pig', 'hippo', 'elephant' , 'tiger', 'deer',
+            'mouse', 'fox', 'horse', 'pig', 'hippo', 'elephant' , 'tiger', 'deer', 'salmon',
             'gerbil', 'snake', 'turtle', 'rhino', 'dog', 'cat', 'giraffe', 'rabbit', 'squirrel',
             'unicorn', 'lizard', 'lion', 'bear', 'gazelle', 'whale', 'dolphin', 'fish', 'butterfly',
             'ladybug', 'fly', 'shrimp', 'flamingo', 'parrot', 'tuna', 'panda', 'lemur', 'duck',
             'seal', 'walrus', 'seagull', 'iguana', 'salamander', 'kitten', 'puppy', 'octopus',
         ],
+        'weather': [
+            'rain', 'sun', 'snow', 'wind', 'tornado', 'hurricane', 'blizzard', 'monsoon', 'storm',
+            'shower', 'hail',
+        ],
         'plants': [
             'tree', 'flower', 'vine', 'fern', 'palm', 'palmetto', 'oak', 'pine', 'rose', 'lily',
-            'ivy',
+            'ivy', 'leaf', 'shrubbery', 'acorn', 'fruit',
         ],
         'foods': [
             'pizza', 'sushi', 'apple', 'banana', 'sandwich', 'burger', 'taco', 'bratwurst',
@@ -74,8 +82,6 @@ _bank: Dict[str, Dict[str, List[str]]] = {
     },
 }
 
-_disallow_combinations: List[Tuple[str, str]] = []
-
 _adjectives: List[str]= []
 for category, items in _bank['adjectives'].items():
     _adjectives += items
@@ -96,12 +102,8 @@ def generate_random_name(separator: str = '-'):
     -------
     A string containing an random adjective and random noun. 
     """
-    while True:
-        adjective_category = random.choice(list(_bank['adjectives'].keys()))
-        noun_category = random.choice(list(_bank['nouns'].keys()))
-        if (adjective_category, noun_category) in _disallow_combinations:
-            continue
-        break
+    adjective_category = random.choice(list(_bank['adjectives'].keys()))
+    noun_category = random.choice(list(_bank['nouns'].keys()))
     return (
         random.choice(_bank['adjectives'][adjective_category])
         + separator

{meerschaum-2.3.0rc3.dist-info → meerschaum-2.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.3.0rc3
+Version: 2.3.1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares