meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/__init__.py

@@ -20,15 +20,17 @@ limitations under the License.
 
 import atexit
 from meerschaum.utils.typing import SuccessTuple
+from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
+from meerschaum.jobs import Job, make_executor
 from meerschaum.connectors import get_connector, Connector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
 from meerschaum.config import __version__, get_config
-from meerschaum.utils.packages import attempt_import
+from meerschaum._internal.entry import entry
 from meerschaum.__main__ import _close_pools
 
 atexit.register(_close_pools)
@@ -42,14 +44,17 @@ __all__ = (
     "Plugin",
     "Venv",
     "Plugin",
+    "Job",
     "pprint",
     "attempt_import",
     "actions",
     "config",
     "connectors",
+    "jobs",
     "plugins",
     "utils",
     "SuccessTuple",
     "Connector",
     "make_connector",
+    "entry",
 )

meerschaum/__main__.py

@@ -19,9 +19,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import sys, os, copy
+import sys
+import os
+import copy
 
-def main(sysargs: list = None) -> None:
+from meerschaum.utils.typing import List, Optional
+from meerschaum.utils.formatting import print_tuple as _print_tuple
+
+
+def main(sysargs: Optional[List[str]] = None) -> None:
     """Main CLI entry point."""
     if sysargs is None:
         sysargs = copy.deepcopy(sys.argv[1:])
@@ -39,11 +45,6 @@ def main(sysargs: list = None) -> None:
         parse_version(sysargs)
         return _exit(old_cwd=old_cwd)
 
-    if ('-d' in sysargs or '--daemon' in sysargs) and ('stack' not in sysargs):
-        from meerschaum.utils.daemon import daemon_entry
-        daemon_entry(sysargs)
-        return _exit(old_cwd=old_cwd)
-
     from meerschaum._internal.entry import entry, get_shell
 
     ### Try to launch a shell if --shell is provided.
@@ -57,8 +58,7 @@ def main(sysargs: list = None) -> None:
     return_tuple = entry(sysargs)
     rc = 0
     if isinstance(return_tuple, tuple) and '--nopretty' not in sysargs:
-        from meerschaum.utils.formatting import print_tuple
-        print_tuple(return_tuple, upper_padding=1)
+        _print_tuple(return_tuple, upper_padding=1)
         rc = 0 if (return_tuple[0] is True) else 1
 
     return _exit(rc, old_cwd=old_cwd)

meerschaum/_internal/arguments/__init__.py

@@ -8,7 +8,7 @@ This package includes argument parsing utilities.
 
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
-    parse_dict_to_sysargs,
+    parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

meerschaum/_internal/arguments/_parse_arguments.py

@@ -10,7 +10,7 @@ This module contains functions for parsing arguments
 from __future__ import annotations
 import json
 from datetime import timedelta
-from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple
+from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple, Tuple
 from meerschaum.utils.threading import RLock
 
 _locks = {
@@ -18,6 +18,59 @@ _locks = {
 }
 _loaded_plugins_args: bool = False
 
+def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
+    """
+    Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
+    if pipeline_key not in sysargs:
+        return sysargs, []
+
+    ### Find the index of the last occurrence of `:`.
+    pipeline_ix = len(sysargs) - 1 - sysargs[::-1].index(pipeline_key)
+    sysargs_after_pipeline_key = sysargs[pipeline_ix+1:]
+    sysargs = [arg for arg in sysargs[:pipeline_ix] if arg != pipeline_key]
+    return sysargs, sysargs_after_pipeline_key
+
+
+def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
+    """
+    Split a `sysargs` list containing "and" keys (`+`)
+    into a list of individual `sysargs`.
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    and_key = STATIC_CONFIG['system']['arguments']['and_key']
+
+    if not sysargs or and_key not in sysargs:
+        return [sysargs]
+
+    ### Coalesce and consecutive joiners into one.
+    coalesce_args = []
+    previous_arg = None
+    for arg in [_arg for _arg in sysargs]:
+        if arg == and_key and previous_arg == and_key:
+            continue
+        coalesce_args.append(arg)
+        previous_arg = arg
+
+    ### Remove any joiners from the ends.
+    if coalesce_args[0] == and_key:
+        coalesce_args = coalesce_args[1:]
+    if coalesce_args[-1] == and_key:
+        coalesce_args = coalesce_args[:-1]
+
+    chained_sysargs = []
+    current_sysargs = []
+    for arg in coalesce_args:
+        if arg != and_key:
+            current_sysargs.append(arg)
+        else:
+            chained_sysargs.append(current_sysargs)
+            current_sysargs = []
+    chained_sysargs.append(current_sysargs)
+    return chained_sysargs
+
 
 def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
     """
@@ -206,9 +259,14 @@ def parse_dict_to_sysargs(
     args_dict: Dict[str, Any]
 ) -> List[str]:
     """Revert an arguments dictionary back to a command line list."""
+    import shlex
     from meerschaum._internal.arguments._parser import get_arguments_triggers
-    sysargs = []
-    sysargs += args_dict.get('action', [])
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.warnings import warn
+
+    action = args_dict.get('action', None)
+    sysargs: List[str] = []
+    sysargs.extend(action or [])
     allow_none_args = {'location_keys'}
 
     triggers = get_arguments_triggers()
@@ -216,6 +274,7 @@ def parse_dict_to_sysargs(
     for a, t in triggers.items():
         if a == 'action' or a not in args_dict:
             continue
+
         ### Add boolean flags
         if isinstance(args_dict[a], bool):
             if args_dict[a] is True:
@@ -288,9 +347,6 @@ def remove_leading_action(
     for a in action:
         _action.append(a.replace('_', UNDERSCORE_STANDIN))
 
-    ### e.g. 'show_pipes_baz'
-    action_str = '_'.join(_action)
-
     ### e.g. 'show_pipes'
     action_name = action_function.__name__.lstrip('_')
 
@@ -300,6 +356,16 @@ def remove_leading_action(
     ### Strip away any leading prefices.
     action_name = action_name[main_action_index:]
 
+    subaction_parts = action_name.replace(main_action_name, '').lstrip('_').split('_')
+    subaction_name = subaction_parts[0] if subaction_parts else None
+
+    ### e.g. 'pipe' -> 'pipes'
+    if subaction_name and subaction_name.endswith('s') and not action[1].endswith('s'):
+        _action[1] += 's'
+
+    ### e.g. 'show_pipes_baz'
+    action_str = '_'.join(_action)
+
     if not action_str.replace(UNDERSCORE_STANDIN, '_').startswith(action_name):
         warn(f"Unable to parse '{action_str}' for action '{action_name}'.")
         return action

meerschaum/_internal/arguments/_parser.py

@@ -62,7 +62,24 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     return dt
 
 
-def parse_help(sysargs : Union[List[str], Dict[str, Any]]) -> None:
+def parse_executor_keys(executor_keys_str: str) -> Union[str, None]:
+    """
+    Ensure that only API keys are provided for executor_keys.
+    """
+    if executor_keys_str in ('local', 'systemd'):
+        return executor_keys_str
+
+    if executor_keys_str.lower() == 'none':
+        return 'local'
+
+    if not executor_keys_str.startswith('api:'):
+        from meerschaum.utils.warnings import error
+        error(f"Invalid exectutor keys '{executor_keys_str}'.", stack=False)
+
+    return executor_keys_str
+
+
+def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     """Parse the `--help` flag to determine which help message to print."""
     from meerschaum._internal.arguments._parse_arguments import parse_arguments, parse_line
     from meerschaum.actions import actions, get_subactions
@@ -135,6 +152,7 @@ _seen_plugin_args = {}
 
 groups = {}
 groups['actions'] = parser.add_argument_group(title='Actions options')
+groups['jobs'] = parser.add_argument_group(title='Jobs options')
 groups['pipes'] = parser.add_argument_group(title='Pipes options')
 groups['sync'] = parser.add_argument_group(title='Sync options')
 groups['api'] = parser.add_argument_group(title='API options')
@@ -166,18 +184,25 @@ groups['actions'].add_argument(
     help="Automatically choose the defaults answers to questions. Does not result in data loss.",
 )
 groups['actions'].add_argument(
-    '-d', '--daemon', action='store_true',
-    help = "Run an action as a background daemon."
+    '-A', '--sub-args', nargs=argparse.REMAINDER,
+    help = (
+        "Provide a list of arguments for subprocesses. " +
+        "You can also type sub-arguments in [] instead." +
+        " E.g. `stack -A='--version'`, `ls [-lh]`, `echo -A these are sub-arguments`"
+    )
 )
-groups['actions'].add_argument(
-    '--rm', action='store_true', help="Delete a job once it has finished executing."
+
+### Jobs options
+groups['jobs'].add_argument(
+    '-d', '--daemon', action='store_true',
+    help = "Run an action as a background daemon to create a job."
 )
-groups['actions'].add_argument(
+groups['jobs'].add_argument(
     '--name', '--job-name', type=parse_name, help=(
         "Assign a name to a job. If no name is provided, a random name will be assigned."
     ),
 )
-groups['actions'].add_argument(
+groups['jobs'].add_argument(
     '-s', '--schedule', '--cron', type=str,
     help = (
         "Continue executing the action according to a schedule (e.g. 'every 1 seconds'). \n"
@@ -185,15 +210,20 @@ groups['actions'].add_argument(
         + "https://red-engine.readthedocs.io/en/stable/condition_syntax/index.html"
     )
 )
-groups['actions'].add_argument(
-    '-A', '--sub-args', nargs=argparse.REMAINDER,
-    help = (
-        "Provide a list of arguments for subprocesses. " +
-        "You can also type sub-arguments in [] instead." +
-        " E.g. `stack -A='--version'`, `ls [-lh]`, `echo -A these are sub-arguments`"
-    )
+groups['jobs'].add_argument(
+    '--restart', action='store_true',
+    help=("Restart a job if not stopped manually."),
+)
+groups['jobs'].add_argument(
+    '-e', '--executor-keys', type=parse_executor_keys,
+    help=(
+        "Execute jobs locally or remotely. "
+        "Supported values are 'local', 'systemd', and 'api:{label}'."
+    ),
+)
+groups['jobs'].add_argument(
+    '--rm', action='store_true', help="Delete a job once it has finished executing."
 )
-
 ### Pipes options
 groups['pipes'].add_argument(
     '-c', '-C', '--connector-keys', nargs='+',

meerschaum/_internal/docs/index.py

@@ -36,8 +36,9 @@ For your convenience, the following classes and functions may be imported from t
 <li><code>meerschaum.Connector</code></li>
 <li><code>meerschaum.Pipe</code></li>
 <li><code>meerschaum.Plugin</code></li>
-<li><code>meerschaum.SuccessTuple</code></li>
+<li><code>meerschaum.Job</code></li>
 <li><code>meerschaum.Venv</code></li>
+<li><code>meerschaum.SuccessTuple</code></li>
 </ul>
 
 </div>
@@ -54,6 +55,7 @@ For your convenience, the following classes and functions may be imported from t
 <li><code>meerschaum.make_connector()</code></li>
 <li><code>meerschaum.pprint()</code></li>
 <li><code>meerschaum.attempt_import()</code></li>
+<li><code>meerschaum.entry()</code></li>
 </ul>
 
 </div>
@@ -61,7 +63,10 @@ For your convenience, the following classes and functions may be imported from t
 
 ### Examples
 
-#### Build a Connector
+<details>
+    <summary><b>Build a Connector</b></summary>
+
+Get existing connectors or build a new one in-memory with the `meerschaum.get_connector()` factory function:
 
 ```python
 import meerschaum as mrsm
@@ -80,9 +85,14 @@ sql_conn.to_sql(df, 'foo')
 print(sql_conn.read('foo'))
 #    foo
 # 0    1
+
 ```
+</details>
+
+<details>
+    <summary><b>Create a Custom Connector Class</b></summary>
 
-#### Create a Custom Connector Class
+Decorate your connector classes with `meerschaum.make_connector()` to designate it as a custom connector:
 
 ```python
 from datetime import datetime, timezone
@@ -113,8 +123,12 @@ foo_conn = mrsm.get_connector(
 )
 docs = foo_conn.fetch()
 ```
+</details>
 
-#### Build a Pipe
+<details>
+    <summary><b>Build a Pipe</b></summary>
+
+Build a `meerschaum.Pipe` in-memory:
 
 ```python
 from datetime import datetime
@@ -135,7 +149,23 @@ print(df)
 # 2 2024-01-01   3  96
 ```
 
-#### Get Registered Pipes
+Add `temporary=True` to skip registering the pipe in the pipes table.
+
+</details>
+
+<details>
+    <summary><b>Get Registered Pipes</b></summary>
+
+The `meerschaum.get_pipes()` function returns a dictionary hierarchy of pipes by connector, metric, and location:
+
+```python
+import meerschaum as mrsm
+
+pipes = mrsm.get_pipes(instance='sql:temp')
+pipe = pipes['foo:bar']['demo'][None]
+```
+
+Add `as_list=True` to flatten the hierarchy:
 
 ```python
 import meerschaum as mrsm
@@ -148,8 +178,12 @@ pipes = mrsm.get_pipes(
 print(pipes)
 # [Pipe('foo:bar', 'demo', instance='sql:temp')]
 ```
+</details>
 
-#### Access a Plugin's Module
+<details>
+    <summary><b>Import Plugins</b></summary>
+
+You can import a plugin's module through `meerschaum.Plugin.module`:
 
 ```python
 import meerschaum as mrsm
@@ -157,10 +191,211 @@ import meerschaum as mrsm
 plugin = mrsm.Plugin('noaa')
 with mrsm.Venv(plugin):
     noaa = plugin.module
-    print(noaa.get_station_info('KGMU'))
-# {'name': 'Greenville Downtown Airport', 'geometry': {'type': 'Point', 'coordinates': [-82.35004, 34.84873]}}
 ```
 
+If your plugin has submodules, use `meerschaum.plugins.from_plugin_import`:
+
+```python
+from meerschaum.plugins import from_plugin_import
+get_defined_pipes = from_plugin_import('compose.utils.pipes', 'get_defined_pipes')
+```
+
+Import multiple plugins with `meerschaum.plugins.import_plugins`:
+
+```python
+from meerschaum.plugins import import_plugins
+noaa, compose = import_plugins('noaa', 'compose')
+```
+
+</details>
+
+<details>
+    <summary><b>Create a Job</b></summary>
+
+Create a `meerschaum.Job` with `name` and `sysargs`:
+
+```python
+import meerschaum as mrsm
+
+job = mrsm.Job('syncing-engine', 'sync pipes --loop')
+success, msg = job.start()
+```
+
+Pass `executor_keys` as the connectors keys of an API instance to create a remote job:
+
+```python
+import meerschaum as mrsm
+
+job = mrsm.Job(
+    'foo',
+    'sync pipes -s daily',
+    executor_keys='api:main',
+)
+```
+
+</details>
+
+<details>
+    <summary><b>Import from a Virtual Environment</b></summary>
+Use the `meerschaum.Venv` context manager to activate a virtual environment:
+```python
+import meerschaum as mrsm
+
+with mrsm.Venv('noaa'):
+    import requests
+
+print(requests.__file__)
+# /home/bmeares/.config/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py
+```
+
+To import packages which may not be installed, use `meerschaum.attempt_import()`:
+
+```python
+import meerschaum as mrsm
+
+requests = mrsm.attempt_import('requests', venv='noaa')
+print(requests.__file__)
+# /home/bmeares/.config/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py
+```
+
+</details>
+
+<details>
+    <summary><b>Run Actions</b></summary>
+
+Run `sysargs` with `meerschaum.entry()`:
+
+```python
+import meerschaum as mrsm
+
+success, msg = mrsm.entry('show pipes + show version : x2')
+```
+
+Use `meerschaum.actions.get_action()` to access an action function directly:
+
+```python
+from meerschaum.actions import get_action
+
+show_pipes = get_action(['show', 'pipes'])
+success, msg = show_pipes(connector_keys=['plugin:noaa'])
+```
+
+Get a dictionary of available subactions with `meerschaum.actions.get_subactions()`:
+
+```python
+from meerschaum.actions import get_subactions
+
+subactions = get_subactions('show')
+success, msg = subactions['pipes']()
+```
+
+</details>
+
+<details>
+    <summary><b>Create a Plugin</b></summary>
+
+Run `bootstrap plugin` to create a new plugin:
+
+```
+mrsm bootstrap plugin example
+```
+
+This will create `example.py` in your plugins directory (default `~/.config/meerschaum/plugins/`, Windows: `%APPDATA%\Meerschaum\plugins`). You may paste the example code from the "Create a Custom Action" example below.
+
+Open your plugin with `edit plugin`:
+
+```
+mrsm edit plugin example
+```
+
+*Run `edit plugin` and paste the example code below to try out the features.*
+
+See the [writing plugins guide](https://meerschaum.io/reference/plugins/writing-plugins/) for more in-depth documentation.
+
+</details>
+
+<details>
+    <summary><b>Create a Custom Action</b></summary>
+
+Decorate a function with `meerschaum.actions.make_action` to designate it as an action. Subactions will be automatically detected if not decorated:
+
+```python
+from meerschaum.actions import make_action
+
+@make_action
+def sing():
+    print('What would you like me to sing?')
+    return True, "Success"
+
+def sing_tune():
+    return False, "I don't know that song!"
+
+def sing_song():
+    print('Hello, World!')
+    return True, "Success"
+
+```
+
+Use `meerschaum.plugins.add_plugin_argument()` to create new parameters for your action:
+
+```python
+from meerschaum.plugins import make_action, add_plugin_argument
+
+add_plugin_argument(
+    '--song', type=str, help='What song to sing.',
+)
+
+@make_action
+def sing_melody(action=None, song=None):
+    to_sing = action[0] if action else song
+    if not to_sing:
+        return False, "Please tell me what to sing!"
+
+    return True, f'~I am singing {to_sing}~'
+```
+
+```
+mrsm sing melody lalala
+
+mrsm sing melody --song do-re-mi
+```
+
+</details>
+
+<details>
+    <summary><b>Add a Page to the Web Dashboard</b></summary>
+    Use the decorators `meerschaum.plugins.dash_plugin()` and `meerschaum.plugins.web_page()` to add new pages to the web dashboard:
+
+```python
+from meerschaum.plugins import dash_plugin, web_page
+
+@dash_plugin
+def init_dash(dash_app):
+
+    import dash.html as html
+    import dash_bootstrap_components as dbc
+    from dash import Input, Output, no_update
+
+    ### Routes to '/dash/my-page'
+    @web_page('/my-page', login_required=False)
+    def my_page():
+        return dbc.Container([
+            html.H1("Hello, World!"),
+            dbc.Button("Click me", id='my-button'),
+            html.Div(id="my-output-div"),
+        ])
+
+    @dash_app.callback(
+        Output('my-output-div', 'children'),
+        Input('my-button', 'n_clicks'),
+    )
+    def my_button_click(n_clicks):
+        if not n_clicks:
+            return no_update
+        return html.P(f'You clicked {n_clicks} times!')
+```
+</details>
+
 ## Submodules
 
 <details>
@@ -206,6 +441,28 @@ with mrsm.Venv(plugin):
 
 </details>
 
+<details>
+    <summary>
+    `meerschaum.jobs`<br>
+    Start background jobs.
+    </summary>
+
+  - `meerschaum.jobs.Job`
+  - `meerschaum.jobs.Executor`
+  - `meerschaum.jobs.systemd.SystemdExecutor`
+  - `meerschaum.jobs.get_jobs()`
+  - `meerschaum.jobs.get_filtered_jobs()`
+  - `meerschaum.jobs.get_running_jobs()`
+  - `meerschaum.jobs.get_stopped_jobs()`
+  - `meerschaum.jobs.get_paused_jobs()`
+  - `meerschaum.jobs.get_restart_jobs()`
+  - `meerschaum.jobs.make_executor()`
+  - `meerschaum.jobs.check_restart_jobs()`
+  - `meerschaum.jobs.start_check_jobs_thread()`
+  - `meerschaum.jobs.stop_check_jobs_thread()`
+
+</details>
+
 <details>
   <summary>
   `meerschaum.plugins`<br>