meerschaum 2.3.0rc3__py3-none-any.whl → 2.3.1__py3-none-any.whl

meerschaum/__init__.py

@@ -24,7 +24,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
-from meerschaum.jobs import Job, Executor, make_executor
+from meerschaum.jobs import Job, make_executor
 from meerschaum.connectors import get_connector, Connector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
@@ -45,13 +45,12 @@ __all__ = (
     "Venv",
     "Plugin",
     "Job",
-    "Executor",
-    "make_executor",
     "pprint",
     "attempt_import",
     "actions",
     "config",
     "connectors",
+    "jobs",
     "plugins",
     "utils",
     "SuccessTuple",

meerschaum/_internal/arguments/__init__.py

@@ -8,7 +8,7 @@ This package includes argument parsing utilities.
 
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
-    parse_dict_to_sysargs, split_chained_sysargs,
+    parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

meerschaum/_internal/arguments/_parse_arguments.py

@@ -10,7 +10,7 @@ This module contains functions for parsing arguments
 from __future__ import annotations
 import json
 from datetime import timedelta
-from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple
+from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple, Tuple
 from meerschaum.utils.threading import RLock
 
 _locks = {
@@ -18,10 +18,25 @@ _locks = {
 }
 _loaded_plugins_args: bool = False
 
+def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
+    """
+    Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
+    if pipeline_key not in sysargs:
+        return sysargs, []
+
+    ### Find the index of the last occurrence of `:`.
+    pipeline_ix = len(sysargs) - 1 - sysargs[::-1].index(pipeline_key)
+    sysargs_after_pipeline_key = sysargs[pipeline_ix+1:]
+    sysargs = [arg for arg in sysargs[:pipeline_ix] if arg != pipeline_key]
+    return sysargs, sysargs_after_pipeline_key
+
 
 def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
     """
-    Split a `sysargs` list containing "and" keys (`&&`)
+    Split a `sysargs` list containing "and" keys (`+`)
     into a list of individual `sysargs`.
     """
     from meerschaum.config.static import STATIC_CONFIG
@@ -91,8 +106,8 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
     for i, word in enumerate(sysargs):
         is_sub_arg = False
         if not found_begin_decorator:
-            found_begin_decorator = word.startswith(begin_decorator)
-            found_end_decorator = word.endswith(end_decorator)
+            found_begin_decorator = str(word).startswith(begin_decorator)
+            found_end_decorator = str(word).endswith(end_decorator)
 
         if found_begin_decorator:
             ### check if sub arg is ever closed
@@ -102,15 +117,15 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
                     found_begin_decorator = False
         elif found_end_decorator:
             for a in sysargs[:i]:
-                if a.startswith(begin_decorator):
+                if str(a).startswith(begin_decorator):
                     is_sub_arg = True
                     found_begin_decorator = False
         if is_sub_arg:
             ### remove decorators
             sa = word
-            if sa.startswith(begin_decorator):
+            if str(sa).startswith(begin_decorator):
                 sa = sa[len(begin_decorator):]
-            if sa.endswith(end_decorator):
+            if str(sa).endswith(end_decorator):
                 sa = sa[:-1 * len(end_decorator)]
             sub_arguments.append(sa)
             ### remove sub-argument from action list
@@ -129,7 +144,7 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
     except Exception as e:
         _action = []
         for a in filtered_sysargs:
-            if a.startswith('-'):
+            if str(a).startswith('-'):
                 break
             _action.append(a)
         args_dict = {'action': _action, 'sysargs': sysargs}
@@ -249,20 +264,10 @@ def parse_dict_to_sysargs(
     from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
 
-    and_key = STATIC_CONFIG['system']['arguments']['and_key']
-    if (line := args_dict.get('line', None)):
-        return shlex.split(line)
-
-    if (_sysargs := args_dict.get('sysargs', None)):
-        return _sysargs
-
     action = args_dict.get('action', None)
-    if action and and_key in action:
-        warn(f"Cannot determine flags from chained actions:\n{args_dict}")
-
     sysargs: List[str] = []
     sysargs.extend(action or [])
-    allow_none_args = {'location_keys'}
+    allow_none_args = {'location_keys', 'begin', 'end', 'executor_keys'}
 
     triggers = get_arguments_triggers()
 
@@ -273,12 +278,18 @@ def parse_dict_to_sysargs(
         ### Add boolean flags
         if isinstance(args_dict[a], bool):
             if args_dict[a] is True:
-                sysargs += [t[0]]
+                sysargs.extend([t[0]])
         else:
             ### Add list flags
             if isinstance(args_dict[a], (list, tuple)):
                 if len(args_dict[a]) > 0:
-                    sysargs += [t[0]] + list(args_dict[a])
+                    sysargs.extend(
+                        [t[0]]
+                        + [
+                            str(item)
+                            for item in args_dict[a]
+                        ]
+                    )
 
             ### Add dict flags
             elif isinstance(args_dict[a], dict):
@@ -287,7 +298,7 @@ def parse_dict_to_sysargs(
 
             ### Account for None and other values
             elif (args_dict[a] is not None) or (args_dict[a] is None and a in allow_none_args):
-                sysargs += [t[0], args_dict[a]]
+                sysargs += [t[0], str(args_dict[a])]
 
     return sysargs
 

meerschaum/_internal/arguments/_parser.py

@@ -125,7 +125,7 @@ def parse_version(sysargs: List[str]):
     return print(doc)
 
 
-def parse_name(name_str: str) -> str:
+def parse_name(name_str: str) -> Union[str, None]:
     """
     Ensure that `--name` is not an empty string.
     """
@@ -215,13 +215,10 @@ groups['jobs'].add_argument(
     help=("Restart a job if not stopped manually."),
 )
 groups['jobs'].add_argument(
-    '--systemd', action='store_true',
-    help=("Create a job via systemd. Shorthand for `-e systemd`."),
-)
-groups['jobs'].add_argument(
-    '--executor-keys', '--executor', '-e', type=parse_executor_keys,
+    '-e', '--executor-keys', type=parse_executor_keys,
     help=(
-        "Execute jobs on an API instance or via systemd."
+        "Execute jobs locally or remotely. "
+        "Supported values are 'local', 'systemd', and 'api:{label}'."
     ),
 )
 groups['jobs'].add_argument(

meerschaum/_internal/docs/index.py

@@ -36,8 +36,9 @@ For your convenience, the following classes and functions may be imported from t
 <li><code>meerschaum.Connector</code></li>
 <li><code>meerschaum.Pipe</code></li>
 <li><code>meerschaum.Plugin</code></li>
-<li><code>meerschaum.SuccessTuple</code></li>
+<li><code>meerschaum.Job</code></li>
 <li><code>meerschaum.Venv</code></li>
+<li><code>meerschaum.SuccessTuple</code></li>
 </ul>
 
 </div>
@@ -54,6 +55,7 @@ For your convenience, the following classes and functions may be imported from t
 <li><code>meerschaum.make_connector()</code></li>
 <li><code>meerschaum.pprint()</code></li>
 <li><code>meerschaum.attempt_import()</code></li>
+<li><code>meerschaum.entry()</code></li>
 </ul>
 
 </div>
@@ -61,7 +63,10 @@ For your convenience, the following classes and functions may be imported from t
 
 ### Examples
 
-#### Build a Connector
+<details>
+    <summary><b>Build a Connector</b></summary>
+
+Get existing connectors or build a new one in-memory with the `meerschaum.get_connector()` factory function:
 
 ```python
 import meerschaum as mrsm
@@ -80,9 +85,14 @@ sql_conn.to_sql(df, 'foo')
 print(sql_conn.read('foo'))
 #    foo
 # 0    1
+
 ```
+</details>
+
+<details>
+    <summary><b>Create a Custom Connector Class</b></summary>
 
-#### Create a Custom Connector Class
+Decorate your connector classes with `meerschaum.make_connector()` to designate it as a custom connector:
 
 ```python
 from datetime import datetime, timezone
@@ -113,8 +123,12 @@ foo_conn = mrsm.get_connector(
 )
 docs = foo_conn.fetch()
 ```
+</details>
 
-#### Build a Pipe
+<details>
+    <summary><b>Build a Pipe</b></summary>
+
+Build a `meerschaum.Pipe` in-memory:
 
 ```python
 from datetime import datetime
@@ -135,7 +149,23 @@ print(df)
 # 2 2024-01-01   3  96
 ```
 
-#### Get Registered Pipes
+Add `temporary=True` to skip registering the pipe in the pipes table.
+
+</details>
+
+<details>
+    <summary><b>Get Registered Pipes</b></summary>
+
+The `meerschaum.get_pipes()` function returns a dictionary hierarchy of pipes by connector, metric, and location:
+
+```python
+import meerschaum as mrsm
+
+pipes = mrsm.get_pipes(instance='sql:temp')
+pipe = pipes['foo:bar']['demo'][None]
+```
+
+Add `as_list=True` to flatten the hierarchy:
 
 ```python
 import meerschaum as mrsm
@@ -148,8 +178,12 @@ pipes = mrsm.get_pipes(
 print(pipes)
 # [Pipe('foo:bar', 'demo', instance='sql:temp')]
 ```
+</details>
 
-#### Access a Plugin's Module
+<details>
+    <summary><b>Import Plugins</b></summary>
+
+You can import a plugin's module through `meerschaum.Plugin.module`:
 
 ```python
 import meerschaum as mrsm
@@ -157,10 +191,211 @@ import meerschaum as mrsm
 plugin = mrsm.Plugin('noaa')
 with mrsm.Venv(plugin):
     noaa = plugin.module
-    print(noaa.get_station_info('KGMU'))
-# {'name': 'Greenville Downtown Airport', 'geometry': {'type': 'Point', 'coordinates': [-82.35004, 34.84873]}}
 ```
 
+If your plugin has submodules, use `meerschaum.plugins.from_plugin_import`:
+
+```python
+from meerschaum.plugins import from_plugin_import
+get_defined_pipes = from_plugin_import('compose.utils.pipes', 'get_defined_pipes')
+```
+
+Import multiple plugins with `meerschaum.plugins.import_plugins`:
+
+```python
+from meerschaum.plugins import import_plugins
+noaa, compose = import_plugins('noaa', 'compose')
+```
+
+</details>
+
+<details>
+    <summary><b>Create a Job</b></summary>
+
+Create a `meerschaum.Job` with `name` and `sysargs`:
+
+```python
+import meerschaum as mrsm
+
+job = mrsm.Job('syncing-engine', 'sync pipes --loop')
+success, msg = job.start()
+```
+
+Pass `executor_keys` as the connectors keys of an API instance to create a remote job:
+
+```python
+import meerschaum as mrsm
+
+job = mrsm.Job(
+    'foo',
+    'sync pipes -s daily',
+    executor_keys='api:main',
+)
+```
+
+</details>
+
+<details>
+    <summary><b>Import from a Virtual Environment</b></summary>
+Use the `meerschaum.Venv` context manager to activate a virtual environment:
+```python
+import meerschaum as mrsm
+
+with mrsm.Venv('noaa'):
+    import requests
+
+print(requests.__file__)
+# /home/bmeares/.config/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py
+```
+
+To import packages which may not be installed, use `meerschaum.attempt_import()`:
+
+```python
+import meerschaum as mrsm
+
+requests = mrsm.attempt_import('requests', venv='noaa')
+print(requests.__file__)
+# /home/bmeares/.config/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py
+```
+
+</details>
+
+<details>
+    <summary><b>Run Actions</b></summary>
+
+Run `sysargs` with `meerschaum.entry()`:
+
+```python
+import meerschaum as mrsm
+
+success, msg = mrsm.entry('show pipes + show version : x2')
+```
+
+Use `meerschaum.actions.get_action()` to access an action function directly:
+
+```python
+from meerschaum.actions import get_action
+
+show_pipes = get_action(['show', 'pipes'])
+success, msg = show_pipes(connector_keys=['plugin:noaa'])
+```
+
+Get a dictionary of available subactions with `meerschaum.actions.get_subactions()`:
+
+```python
+from meerschaum.actions import get_subactions
+
+subactions = get_subactions('show')
+success, msg = subactions['pipes']()
+```
+
+</details>
+
+<details>
+    <summary><b>Create a Plugin</b></summary>
+
+Run `bootstrap plugin` to create a new plugin:
+
+```
+mrsm bootstrap plugin example
+```
+
+This will create `example.py` in your plugins directory (default `~/.config/meerschaum/plugins/`, Windows: `%APPDATA%\Meerschaum\plugins`). You may paste the example code from the "Create a Custom Action" example below.
+
+Open your plugin with `edit plugin`:
+
+```
+mrsm edit plugin example
+```
+
+*Run `edit plugin` and paste the example code below to try out the features.*
+
+See the [writing plugins guide](https://meerschaum.io/reference/plugins/writing-plugins/) for more in-depth documentation.
+
+</details>
+
+<details>
+    <summary><b>Create a Custom Action</b></summary>
+
+Decorate a function with `meerschaum.actions.make_action` to designate it as an action. Subactions will be automatically detected if not decorated:
+
+```python
+from meerschaum.actions import make_action
+
+@make_action
+def sing():
+    print('What would you like me to sing?')
+    return True, "Success"
+
+def sing_tune():
+    return False, "I don't know that song!"
+
+def sing_song():
+    print('Hello, World!')
+    return True, "Success"
+
+```
+
+Use `meerschaum.plugins.add_plugin_argument()` to create new parameters for your action:
+
+```python
+from meerschaum.plugins import make_action, add_plugin_argument
+
+add_plugin_argument(
+    '--song', type=str, help='What song to sing.',
+)
+
+@make_action
+def sing_melody(action=None, song=None):
+    to_sing = action[0] if action else song
+    if not to_sing:
+        return False, "Please tell me what to sing!"
+
+    return True, f'~I am singing {to_sing}~'
+```
+
+```
+mrsm sing melody lalala
+
+mrsm sing melody --song do-re-mi
+```
+
+</details>
+
+<details>
+    <summary><b>Add a Page to the Web Dashboard</b></summary>
+    Use the decorators `meerschaum.plugins.dash_plugin()` and `meerschaum.plugins.web_page()` to add new pages to the web dashboard:
+
+```python
+from meerschaum.plugins import dash_plugin, web_page
+
+@dash_plugin
+def init_dash(dash_app):
+
+    import dash.html as html
+    import dash_bootstrap_components as dbc
+    from dash import Input, Output, no_update
+
+    ### Routes to '/dash/my-page'
+    @web_page('/my-page', login_required=False)
+    def my_page():
+        return dbc.Container([
+            html.H1("Hello, World!"),
+            dbc.Button("Click me", id='my-button'),
+            html.Div(id="my-output-div"),
+        ])
+
+    @dash_app.callback(
+        Output('my-output-div', 'children'),
+        Input('my-button', 'n_clicks'),
+    )
+    def my_button_click(n_clicks):
+        if not n_clicks:
+            return no_update
+        return html.P(f'You clicked {n_clicks} times!')
+```
+</details>
+
 ## Submodules
 
 <details>
@@ -206,6 +441,28 @@ with mrsm.Venv(plugin):
 
 </details>
 
+<details>
+    <summary>
+    `meerschaum.jobs`<br>
+    Start background jobs.
+    </summary>
+
+  - `meerschaum.jobs.Job`
+  - `meerschaum.jobs.Executor`
+  - `meerschaum.jobs.systemd.SystemdExecutor`
+  - `meerschaum.jobs.get_jobs()`
+  - `meerschaum.jobs.get_filtered_jobs()`
+  - `meerschaum.jobs.get_running_jobs()`
+  - `meerschaum.jobs.get_stopped_jobs()`
+  - `meerschaum.jobs.get_paused_jobs()`
+  - `meerschaum.jobs.get_restart_jobs()`
+  - `meerschaum.jobs.make_executor()`
+  - `meerschaum.jobs.check_restart_jobs()`
+  - `meerschaum.jobs.start_check_jobs_thread()`
+  - `meerschaum.jobs.stop_check_jobs_thread()`
+
+</details>
+
 <details>
   <summary>
   `meerschaum.plugins`<br>

meerschaum/_internal/entry.py

@@ -32,7 +32,10 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     if _systemd_stdin_path:
         sys.stdin = _StdinFile(_systemd_stdin_path)
 
-def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
+def entry(
+    sysargs: Optional[List[str]] = None,
+    _patch_args: Optional[Dict[str, Any]] = None,
+) -> SuccessTuple:
     """
     Parse arguments and launch a Meerschaum action.
 
@@ -41,14 +44,23 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     A `SuccessTuple` indicating success.
     """
     import shlex
+    import json
     from meerschaum.utils.formatting import make_header
-    from meerschaum._internal.arguments import parse_arguments, split_chained_sysargs
+    from meerschaum._internal.arguments import (
+        parse_arguments,
+        split_chained_sysargs,
+        split_pipeline_sysargs,
+    )
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
         sysargs = []
     if not isinstance(sysargs, list):
         sysargs = shlex.split(sysargs)
 
+    pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
+    escaped_pipeline_key = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
+    sysargs, pipeline_args = split_pipeline_sysargs(sysargs)
+
     has_daemon = '-d' in sysargs or '--daemon' in sysargs
     has_start_job = sysargs[:2] == ['start', 'job']
     chained_sysargs = (
@@ -56,10 +68,32 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
         if has_daemon or has_start_job
         else split_chained_sysargs(sysargs)
     )
+    if pipeline_args:
+        chained_sysargs = [
+            ['start', 'pipeline']
+            + [str(arg) for arg in pipeline_args]
+            + (
+                ['--params', json.dumps(_patch_args)]
+                if _patch_args
+                else []
+            )
+            + ['--sub-args', shlex.join(sysargs)]
+        ]
+
     results: List[SuccessTuple] = []
 
     for _sysargs in chained_sysargs:
+        if escaped_pipeline_key in _sysargs:
+            _sysargs = [
+                pipeline_key
+                if _arg == escaped_pipeline_key
+                else _arg
+                for _arg in _sysargs
+            ]
+
         args = parse_arguments(_sysargs)
+        if _patch_args:
+            args.update(_patch_args)
         argparse_exception = args.get(
             STATIC_CONFIG['system']['arguments']['failure_key'],
             None,
@@ -76,7 +110,7 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
                     )
                 )
 
-        entry_success, entry_msg = entry_with_args(**args)
+        entry_success, entry_msg = entry_with_args(_patch_args=_patch_args, **args)
         if not entry_success:
             return entry_success, entry_msg
 
@@ -106,6 +140,7 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
 def entry_with_args(
     _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+    _patch_args: Optional[Dict[str, Any]] = None,
     **kw
 ) -> SuccessTuple:
     """Execute a Meerschaum action with keyword arguments.
@@ -119,12 +154,15 @@ def entry_with_args(
     from meerschaum.utils.venv import active_venvs, deactivate_venv
     from meerschaum.config.static import STATIC_CONFIG
 
+    if _patch_args:
+        kw.update(_patch_args)
+
     and_key = STATIC_CONFIG['system']['arguments']['and_key']
     escaped_and_key = STATIC_CONFIG['system']['arguments']['escaped_and_key']
     if and_key in (sysargs := kw.get('sysargs', [])):
         if '-d' in sysargs or '--daemon' in sysargs:
             sysargs = [(arg if arg != and_key else escaped_and_key) for arg in sysargs]
-        return entry(sysargs)
+        return entry(sysargs, _patch_args=_patch_args)
 
     if kw.get('trace', None):
         from meerschaum.utils.misc import debug_trace