meerschaum 2.2.5.dev0__py3-none-any.whl → 2.2.5.dev3__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -80,8 +80,9 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 
     ### rebuild sysargs without sub_arguments
     filtered_sysargs = [
-        word for i, word in enumerate(sysargs)
-            if i not in sub_arg_indices
+        word
+        for i, word in enumerate(sysargs)
+        if i not in sub_arg_indices
     ]
 
     try:
@@ -99,7 +100,6 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
         except Exception as _e:
             args_dict['text'] = ' '.join(sysargs)
         args_dict[STATIC_CONFIG['system']['arguments']['failure_key']] = e
-        unknown = []
 
     false_flags = [arg for arg, val in args_dict.items() if val is False]
     for arg in false_flags:
@@ -126,17 +126,26 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 
     ### remove None (but not False) args
     none_args = []
+    none_args_keep = []
     for a, v in args_dict.items():
         if v is None:
             none_args.append(a)
+        elif v == 'None':
+            none_args_keep.append(a)
     for a in none_args:
         del args_dict[a]
+    for a in none_args_keep:
+        args_dict[a] = None
 
     ### location_key '[None]' or 'None' -> None
     if 'location_keys' in args_dict:
         args_dict['location_keys'] = [
-            None if lk in ('[None]', 'None')
-            else lk for lk in args_dict['location_keys'] 
+            (
+                None
+                if lk in ('[None]', 'None')
+                else lk
+            )
+            for lk in args_dict['location_keys']
         ]
 
     return parse_synonyms(args_dict)
@@ -145,7 +154,7 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 def parse_line(line: str) -> Dict[str, Any]:
     """
     Parse a line of text into standard Meerschaum arguments.
-    
+
     Parameters
     ----------
     line: str
@@ -165,12 +174,12 @@ def parse_line(line: str) -> Dict[str, Any]:
     try:
         return parse_arguments(shlex.split(line))
     except Exception as e:
-        return {'action': [], 'text': line,}
+        return {'action': [], 'text': line}
 
 
 def parse_synonyms(
-        args_dict: Dict[str, Any]
-    ) -> Dict[str, Any]:
+    args_dict: Dict[str, Any]
+) -> Dict[str, Any]:
     """Check for synonyms (e.g. `async` = `True` -> `unblock` = `True`)"""
     if args_dict.get('async', None):
         args_dict['unblock'] = True
@@ -194,8 +203,8 @@ def parse_synonyms(
 
 
 def parse_dict_to_sysargs(
-        args_dict : Dict[str, Any]
-    ) -> List[str]:
+    args_dict: Dict[str, Any]
+) -> List[str]:
     """Revert an arguments dictionary back to a command line list."""
     from meerschaum._internal.arguments._parser import get_arguments_triggers
     sysargs = []
@@ -230,9 +239,9 @@ def parse_dict_to_sysargs(
 
 
 def remove_leading_action(
-        action: List[str],
-        _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
-    ) -> List[str]:
+    action: List[str],
+    _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+) -> List[str]:
     """
     Remove the leading strings in the `action` list.
 

meerschaum/_internal/arguments/_parser.py

@@ -37,12 +37,15 @@ class ArgumentParser(argparse.ArgumentParser):
         return result
 
 
-def parse_datetime(dt_str: str) -> datetime:
+def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     """Parse a string into a datetime."""
     from meerschaum.utils.misc import is_int
     if is_int(dt_str):
         return int(dt_str)
 
+    if dt_str in ('None', '[None]'):
+        return 'None'
+
     from meerschaum.utils.packages import attempt_import
     dateutil_parser = attempt_import('dateutil.parser')
 
@@ -155,7 +158,6 @@ groups['actions'].add_argument(
     '-d', '--daemon', action='store_true',
     help = "Run an action as a background daemon."
 )
-
 groups['actions'].add_argument(
     '--rm', action='store_true', help="Delete a job once it has finished executing."
 )

meerschaum/_internal/docs/index.py

@@ -5,130 +5,36 @@
 """
 <img src="https://meerschaum.io/assets/banner_1920x320.png" alt="Meerschaum banner" style="width: 100%;"/>
 
-| PyPI | GitHub | Info | Stats |
-|---|---|---|---|
-| ![PyPI]( https://img.shields.io/pypi/v/meerschaum?color=%2300cc66&label=Version ) | ![GitHub Repo stars](https://img.shields.io/github/stars/bmeares/Meerschaum?style=social) | ![License](https://img.shields.io/github/license/bmeares/Meerschaum?label=License) | ![Number of plugins]( https://img.shields.io/badge/dynamic/json?color=3098c1&label=Public%20Plugins&query=num_plugins&url=https%3A%2F%2Fapi.mrsm.io%2Finfo ) |
-| ![PyPI - Python Version]( https://img.shields.io/pypi/pyversions/meerschaum?label=Python&logo=python&logoColor=%23ffffff ) | ![GitHub Sponsors](https://img.shields.io/github/sponsors/bmeares?color=eadf15&label=Sponsors) | [![meerschaum Tutorials](https://badges.openbase.com/python/tutorials/meerschaum.svg?token=2Yi8Oav9UZYWocO1ncwnIOnpUN5dTnUMWai7lAKTB+k=)](https://openbase.com/python/meerschaum?utm_source=embedded&amp;utm_medium=badge&amp;utm_campaign=rate-badge) | ![Number of registered users]( https://img.shields.io/badge/dynamic/json?color=3098c1&label=Registered%20Users&query=num_users&url=https%3A%2F%2Fapi.mrsm.io%2Finfo ) |
+# Meerschaum Python API
+
+Welcome to the Meerschaum Python API technical documentation!
+
+- `meerschaum.actions`  
+  Access functions for actions and subactions.
+- `meerschaum.connectors`
+- `meerschaum.utils`  
+  Utility functions are available in several submodules, such as functions for:
+  - `meerschaum.utils.daemon`  
+    Managing background processes.\n
+  - `meerschaum.utils.dataframe`  
+    Manipulating dataframes.\n
+  - `meerschaum.utils.dtypes`  
+    Working with data types.\n
+  - `meerschaum.utils.formatting`  
+    Formatting output text.\n
+  - `meerschaum.utils.misc`  
+    Miscellaneous utility functions (e.g. `meerschaum.utils.misc.round_time()`).\n
+  - `meerschaum.utils.packages`  
+    Managing Python packages.\n
+  - `meerschaum.utils.prompt`  
+    Reading input from the user.\n
+  - `meerschaum.utils.schedule`  
+    Scheduling processes and threads.\n
+  - `meerschaum.utils.sql`  
+    Building SQL queries.\n
+  - `meerschaum.utils.venv`  
+    Managing virtual environments.\n
+  - `meerschaum.utils.warnings`  
+    Print warnings, errors, info, and debug messages.
 
-<p align="center">
-<img src="https://meerschaum.io/files/images/demo.gif" alt="Meerschaum demo" style="width: 100%;">
-</p>
-
-## What is Meerschaum?
-Meerschaum is a tool for quickly synchronizing time-series data streams called **pipes**. With Meerschaum, you can have a data visualization stack running in minutes.
-
-<p align="center">
-<img src="https://meerschaum.io/assets/screenshots/weather_pipes.png" style="width: 100%;"/>
-</p>
-
-## Why Meerschaum?
-
-If you've worked with time-series data, you know the headaches that come with ETL.
-Data engineering often gets in analysts' way, and when work needs to get done, every minute spent on pipelining is time taken away from real analysis.
-
-Rather than copy / pasting your ETL scripts, simply build pipes with Meerschaum! [Meerschaum gives you the tools to design your data streams how you like](https://towardsdatascience.com/easy-time-series-etl-for-data-scientists-with-meerschaum-5aade339b398) ― and don't worry — you can always incorporate Meerschaum into your existing systems!
-
-## Features
-
-- 📊 **Built for Data Scientists and Analysts**  
-    - Integrate with Pandas, Grafana and other popular [data analysis tools](https://meerschaum.io/reference/data-analysis-tools/).
-    - Persist your dataframes and always get the latest data.
-- ⚡️ **Production-Ready, Batteries Included**  
-    - [Synchronization engine](https://meerschaum.io/reference/pipes/syncing/) concurrently updates many time-series data streams.
-    - One-click deploy a [TimescaleDB and Grafana stack](https://meerschaum.io/reference/stack/) for prototyping.
-    - Serve data to your entire organization through the power of `uvicorn`, `gunicorn`, and `FastAPI`.
-- 🔌 **Easily Expandable**  
-    -  Ingest any data source with a simple [plugin](https://meerschaum.io/reference/plugins/writing-plugins/). Just return a DataFrame, and Meerschaum handles the rest.
-    - [Add any function as a command](https://meerschaum.io/reference/plugins/types-of-plugins/#action-plugins) to the Meerschaum system.
-    - Include Meerschaum in your projects with its [easy-to-use Python API](https://docs.meerschaum.io).
-- ✨ **Tailored for Your Experience**  
-    - Rich CLI makes managing your data streams surprisingly enjoyable!
-    - Web dashboard for those who prefer a more graphical experience.
-    - Manage your database connections with [Meerschaum connectors](https://meerschaum.io/reference/connectors/)
-    - Utility commands with sensible syntax let you control many pipes with grace.
-- 💼 **Portable from the Start**  
-    - The environment variables `$MRSM_ROOT_DIR`, `$MRSM_PLUGINS_DIR`, and `$MRSM_VENVS_DIR` let you emulate multiple installations and group together your [instances](https://meerschaum.io/reference/connectors/#instances-and-repositories).
-    - No dependencies required; anything needed will be installed into virtual environments.
-    - [Specify required packages for your plugins](https://meerschaum.io/reference/plugins/writing-plugins/), and users will get those packages in a virtual environment.
-
-## Installation
-
-For a more thorough setup guide, visit the [Getting Started](https://meerschaum.io/get-started/) page at [meerschaum.io](https://meerschaum.io).
-
-### TL;DR
-
-```bash
-pip install -U --user meerschaum
-mrsm stack up -d db grafana
-mrsm bootstrap pipes
-```
-
-## Usage Documentation
-
-Please visit [meerschaum.io](https://meerschaum.io) for setup, usage, and troubleshooting information. You can find technical documentation at [docs.meerschaum.io](https://docs.meerschaum.io), and here is a complete list of the [Meerschaum actions](https://meerschaum.io/reference/actions/).
-
-```python
->>> import meerschaum as mrsm
->>> pipe = mrsm.Pipe("plugin:noaa", "weather")
->>> cols_to_select = ['timestamp', 'station', 'temperature (degC)']
->>> df = pipe.get_data(cols_to_select, begin='2023-11-15', end='2023-11-20')
->>> df
-              timestamp station  temperature (degC)
-0   2023-11-15 00:52:00    KATL                16.1
-1   2023-11-15 00:52:00    KCLT                11.7
-2   2023-11-15 00:53:00    KGMU                15.0
-3   2023-11-15 00:54:00    KCEU                13.9
-4   2023-11-15 01:52:00    KATL                15.6
-..                  ...     ...                 ...
-535 2023-11-19 22:54:00    KCEU                15.6
-536 2023-11-19 23:52:00    KATL                16.7
-537 2023-11-19 23:52:00    KCLT                13.9
-538 2023-11-19 23:53:00    KGMU                15.6
-539 2023-11-19 23:54:00    KCEU                15.0
-
-[540 rows x 3 columns]
->>>
-```
-
-## Plugins
-
-Check out the [Awesome Meerschaum list](https://github.com/bmeares/awesome-meerschaum) for a list of community plugins as well as the [public plugins repository](https://api.mrsm.io/dash/plugins).
-
-For details on installing, using, and writing plugins, check out the [plugins documentation](https://meerschaum.io/reference/plugins/types-of-plugins) at [meerschaum.io](https://meerschaum.io).
-
-#### Example Plugin
-
-```python
-# ~/.config/meerschaum/plugins/example.py
-__version__ = '0.0.1'
-required = []
-
-def register(pipe, **kw):
-    return {
-        'columns': {
-            'datetime': 'dt',
-            'id': 'id',
-            'value': 'val',
-        }
-    }
-
-def fetch(pipe, **kw):
-    import random
-    from datetime import datetime
-    docs = [
-        {
-            'dt': datetime.now(),
-            'id': i,
-            'val': random.ranint(0, 200),
-        }
-        for i in range(random.randint(0, 100))
-    ]
-    return docs
-```
-
-## Support Meerschaum's Development
-
-For consulting services and to support Meerschaum's development, please considering sponsoring me on [GitHub sponsors](https://github.com/sponsors/bmeares).
-
-Additionally, you can always [buy me a coffee☕](https://www.buymeacoffee.com/bmeares)!
 """

meerschaum/_internal/shell/Shell.py

@@ -182,7 +182,6 @@ def _check_complete_keys(line: str) -> Optional[List[str]]:
             is_trigger = True
         elif line.endswith(' '):
             ### return empty list so we don't try to parse an incomplete line.
-            #  print('ABORT')
             return []
 
     from meerschaum.utils.misc import get_connector_labels
@@ -196,8 +195,6 @@ def _check_complete_keys(line: str) -> Optional[List[str]]:
             if line.rstrip(' ').endswith(trigger):
                 return get_connector_labels()
 
-            #  args = parse_line(line.rstrip(' '))
-            #  search_term = args[var] if var != 'connector_keys' else args[var][0]
             return get_connector_labels(search_term=last_word.rstrip(' '))
 
     return None

meerschaum/actions/bootstrap.py

@@ -103,12 +103,14 @@ def _bootstrap_pipes(
         )
         try:
             ck = choose(
-                f"Where are the data coming from?\n\n" +
-                f"    Please type the keys of a connector from below,\n" +
-                f"    or enter '{new_label}' to register a new connector.\n\n" +
-                f" {get_config('formatting', 'emoji', 'connector')} Connector:\n",
+                (
+                    "Where are the data coming from?\n\n" +
+                    f"    Please type the keys of a connector or enter '{new_label}'\n" +
+                    "    to register a new connector.\n\n" +
+                    f" {get_config('formatting', 'emoji', 'connector')} Connector:"
+                ),
                 get_connector_labels() + [new_label],
-                numeric = False
+                numeric = False,
             )
         except KeyboardInterrupt:
             return abort_tuple
@@ -258,8 +260,8 @@ def _bootstrap_connectors(
         _type = choose(
             (
                 'Please choose a connector type.\n'
-                + 'For more information on connectors, '
-                + 'please visit https://meerschaum.io/reference/connectors'
+                + '    See https://meerschaum.io/reference/connectors '
+                + 'for documentation on connectors.\n'
             ),
             sorted(list(connectors)),
             default = 'sql'
@@ -391,239 +393,16 @@ def _bootstrap_plugins(
     """
     Launch an interactive wizard to guide the user to creating a new plugin.
     """
-    import pathlib
-    import meerschaum as mrsm
-    from meerschaum.utils.warnings import info, warn
-    from meerschaum.utils.prompt import prompt, choose, yes_no
-    from meerschaum.utils.formatting._shell import clear_screen
-    from meerschaum.utils.misc import edit_file
-    from meerschaum.config.paths import PLUGINS_DIR_PATHS
-    from meerschaum._internal.entry import entry
+    from meerschaum.utils.prompt import prompt
+    from meerschaum.plugins.bootstrap import bootstrap_plugin
 
     if not action:
         action = [prompt("Enter the name of your new plugin:")]
 
-    if len(PLUGINS_DIR_PATHS) > 1:
-        plugins_dir_path = pathlib.Path(
-            choose(
-                "In which directory do you want to write your plugin?",
-                [path.as_posix() for path in PLUGINS_DIR_PATHS],
-                numeric = True,
-                multiple = False,
-                default = PLUGINS_DIR_PATHS[0].as_posix(),
-            )
-        )
-    else:
-        plugins_dir_path = PLUGINS_DIR_PATHS[0]
-        
-    clear_screen(debug=debug)
-    info(
-        "Answer the questions below to pick out features.\n"
-        + "    See the Writing Plugins guide for documentation:\n"
-        + "    https://meerschaum.io/reference/plugins/writing-plugins/ for documentation.\n"
-    )
-
-    imports_lines = {
-        'default': (
-            "import meerschaum as mrsm\n"
-        ),
-        'action': (
-            "from meerschaum.actions import make_action\n"
-        ),
-        'api': (
-            "from meerschaum.plugins import api_plugin\n"
-        ),
-        'web': (
-            "from meerschaum.plugins import web_page, dash_plugin\n"
-        ),
-        'api+web': (
-            "from meerschaum.plugins import api_plugin, web_page, dash_plugin\n"
-        ),
-    }
-
-    ### TODO: Add feature for custom connectors.
-    feature_lines = {
-        'header': (
-            "# {plugin_name}.py\n\n"
-        ),
-        'default': (
-            "__version__ = '0.0.1'\n"
-            "\n# Add any depedencies to `required` (similar to `requirements.txt`).\n"
-            "required = []\n\n"
-        ),
-        'setup': (
-            "def setup(**kwargs) -> mrsm.SuccessTuple:\n"
-            "    \"\"\"Executed during installation and `mrsm setup plugin {plugin_name}`.\"\"\"\n"
-            "    return True, \"Success\"\n\n\n"
-        ),
-        'register': (
-            "def register(pipe: mrsm.Pipe):\n"
-            "    \"\"\"Return the default parameters for a new pipe.\"\"\"\n"
-            "    return {\n"
-            "        'columns': {\n"
-            "            'datetime': None,\n"
-            "        }\n"
-            "    }\n\n\n"
-        ),
-        'fetch': (
-            "def fetch(pipe: mrsm.Pipe, **kwargs):\n"
-            "    \"\"\"Return or yield dataframe-like objects.\"\"\"\n"
-            "    docs = []\n"
-            "    # populate docs with dictionaries (rows).\n"
-            "    return docs\n\n\n"
-        ),
-        'action': (
-            "@make_action\n"
-            "def {action_name}(**kwargs) -> mrsm.SuccessTuple:\n"
-            "    \"\"\"Run `mrsm {action_spaces}` to trigger.\"\"\"\n"
-            "    return True, \"Success\"\n\n\n"
-        ),
-        'api': (
-            "@api_plugin\n"
-            "def init_app(fastapi_app):\n"
-            "    \"\"\"Add new endpoints to the FastAPI app.\"\"\"\n\n"
-            "    import fastapi\n"
-            "    from meerschaum.api import manager\n\n"
-            "    @fastapi_app.get('/my/endpoint')\n"
-            "    def get_my_endpoint(curr_user=fastapi.Depends(manager)):\n"
-            "        return {'message': 'Hello, World!'}\n\n\n"
-        ),
-        'web': (
-            "@dash_plugin\n"
-            "def init_dash(dash_app):\n"
-            "    \"\"\"Initialize the Plotly Dash application.\"\"\"\n"
-            "    import dash.html as html\n"
-            "    import dash.dcc as dcc\n"
-            "    from dash import Input, Output, State, no_update\n"
-            "    import dash_bootstrap_components as dbc\n\n"
-            "    # Create a new page at the path `/dash/{plugin_name}`.\n"
-            "    @web_page('{plugin_name}', login_required=False)\n"
-            "    def page_layout():\n"
-            "        \"\"\"Return the layout objects for this page.\"\"\"\n"
-            "        return dbc.Container([\n"
-            "            dcc.Location(id='{plugin_name}-location'),\n"
-            "            html.Div(id='output-div'),\n"
-            "        ])\n\n"
-            "    @dash_app.callback(\n"
-            "        Output('output-div', 'children'),\n"
-            "        Input('{plugin_name}-location', 'pathname'),\n"
-            "    )\n"
-            "    def render_page_on_url_change(pathname: str):\n"
-            "        \"\"\"Reload page contents when the URL path changes.\"\"\"\n"
-            "        return html.H1(\"Hello from plugin '{plugin_name}'!\")\n\n\n"
-        ),
-    }
-
     for plugin_name in action:
-        plugin_path = plugins_dir_path / (plugin_name + '.py')
-        plugin = mrsm.Plugin(plugin_name)
-        if plugin.is_installed():
-            warn(f"Plugin '{plugin_name}' is already installed!", stack=False)
-            uninstall_plugin = yes_no(
-                f"Do you want to first uninstall '{plugin}'?",
-                default = 'n',
-                **kwargs
-            )
-            if not uninstall_plugin:
-                return False, f"Plugin '{plugin_name}' already exists."
-
-            uninstall_success, uninstall_msg = entry(['uninstall', 'plugin', plugin_name, '-f'])
-            if not uninstall_success:
-                return uninstall_success, uninstall_msg
-            clear_screen(debug=debug)
-
-        features = choose(
-            "Which of the following features would you like to add to your plugin?",
-            [
-                (
-                    'fetch',
-                    'Fetch data\n     (e.g. extracting from a remote API)\n'
-                ),
-                (
-                    'action',
-                    'New actions\n     (e.g. `mrsm sing song`)\n'
-                ),
-                (
-                    'api',
-                    'New API endpoints\n     (e.g. `POST /my/new/endpoint`)\n',
-                ),
-                (
-                    'web',
-                    'New web console page\n     (e.g. `/dash/my-web-app`)\n',
-                ),
-            ],
-            default = 'fetch',
-            multiple = True,
-            as_indices = True,
-            **kwargs
-        )
-
-        action_name = ''
-        if 'action' in features:
-            while True:
-                try:
-                    action_name = prompt(
-                        "What is name of your action?\n    "
-                        + "(separate subactions with spaces, e.g. `sing song`):"
-                    ).replace(' ', '_')
-                except KeyboardInterrupt as e:
-                    return False, "Aborted plugin creation."
-
-                if action_name:
-                    break
-                warn("Please enter an action.", stack=False)
-
-        action_spaces = action_name.replace('_', ' ')
-
-        plugin_labels = {
-            'plugin_name': plugin_name,
-            'action_name': action_name,
-            'action_spaces': action_spaces,
-        }
-
-        body_text = ""
-        body_text += feature_lines['header'].format(**plugin_labels)
-        body_text += imports_lines['default'].format(**plugin_labels)
-        if 'action' in features:
-            body_text += imports_lines['action']
-        if 'api' in features and 'web' in features:
-            body_text += imports_lines['api+web']
-        elif 'api' in features:
-            body_text += imports_lines['api']
-        elif 'web' in features:
-            body_text += imports_lines['web']
-
-        body_text += "\n"
-        body_text += feature_lines['default'].format(**plugin_labels)
-        body_text += feature_lines['setup'].format(**plugin_labels)
-
-        if 'fetch' in features:
-            body_text += feature_lines['register']
-            body_text += feature_lines['fetch']
-
-        if 'action' in features:
-            body_text += feature_lines['action'].format(**plugin_labels)
-
-        if 'api' in features:
-            body_text += feature_lines['api']
-
-        if 'web' in features:
-            body_text += feature_lines['web'].format(**plugin_labels)
-
-        try:
-            with open(plugin_path, 'w+', encoding='utf-8') as f:
-                f.write(body_text.rstrip())
-        except Exception as e:
-            error_msg = f"Failed to write file '{plugin_path}':\n{e}"
-            return False, error_msg
-
-        mrsm.pprint((True, f"Successfully created file '{plugin_path}'."))
-        try:
-            _ = prompt(f"Press [Enter] to edit plugin '{plugin_name}', [CTRL+C] to skip.")
-        except (KeyboardInterrupt, Exception):
-            continue
-
-        edit_file(plugin_path, debug=debug)
+        bootstrap_success, bootstrap_msg = bootstrap_plugin(plugin_name)
+        if not bootstrap_success:
+            return bootstrap_success, bootstrap_msg
 
     return True, "Success"