meerschaum 2.2.5.dev0__py3-none-any.whl → 2.2.5.dev3__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -34,29 +34,29 @@ class InferFetch:
     MRSM_INFER_FETCH: bool = True
 
 def sync(
-        self,
-        df: Union[
-            pd.DataFrame,
-            Dict[str, List[Any]],
-            List[Dict[str, Any]],
-            InferFetch
-        ] = InferFetch,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int] = None,
-        force: bool = False,
-        retries: int = 10,
-        min_seconds: int = 1,
-        check_existing: bool = True,
-        blocking: bool = True,
-        workers: Optional[int] = None,
-        callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
-        error_callback: Optional[Callable[[Exception], Any]] = None,
-        chunksize: Optional[int] = -1,
-        sync_chunks: bool = True,
-        debug: bool = False,
-        _inplace: bool = True,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    df: Union[
+        pd.DataFrame,
+        Dict[str, List[Any]],
+        List[Dict[str, Any]],
+        InferFetch
+    ] = InferFetch,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, None] = None,
+    force: bool = False,
+    retries: int = 10,
+    min_seconds: int = 1,
+    check_existing: bool = True,
+    blocking: bool = True,
+    workers: Optional[int] = None,
+    callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
+    error_callback: Optional[Callable[[Exception], Any]] = None,
+    chunksize: Optional[int] = -1,
+    sync_chunks: bool = True,
+    debug: bool = False,
+    _inplace: bool = True,
+    **kw: Any
+) -> SuccessTuple:
     """
     Fetch new data from the source and update the pipe's table with new data.
     
@@ -125,7 +125,7 @@ def sync(
     from meerschaum.utils.formatting import get_console
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.utils.misc import df_is_chunk_generator
+    from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
     from meerschaum.utils.pool import get_pool
     from meerschaum.config import get_config
 
@@ -186,7 +186,7 @@ def sync(
         ### use that instead.
         ### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
         ### If a DataFrame is provided, continue as expected.
-        if hasattr(df, 'MRSM_INFER_FETCH'):                   
+        if hasattr(df, 'MRSM_INFER_FETCH'):
             try:
                 if p.connector is None:
                     msg = f"{p} does not have a valid connector."
@@ -194,7 +194,7 @@ def sync(
                         msg += f"\n    Perhaps {p.connector_keys} has a syntax error?"
                     p._exists = None
                     return False, msg
-            except Exception as e:
+            except Exception:
                 p._exists = None
                 return False, f"Unable to create the connector for {p}."
 
@@ -210,14 +210,28 @@ def sync(
             ):
                 with Venv(get_connector_plugin(self.instance_connector)):
                     p._exists = None
-                    return self.instance_connector.sync_pipe_inplace(p, debug=debug, **kw)
+                    return self.instance_connector.sync_pipe_inplace(
+                        p,
+                        **filter_keywords(
+                            p.instance_connector.sync_pipe_inplace,
+                            debug=debug,
+                            **kw
+                        )
+                    )
 
 
             ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
             try:
                 if getattr(p.connector, 'sync', None) is not None:
                     with Venv(get_connector_plugin(p.connector), debug=debug):
-                        return_tuple = p.connector.sync(p, debug=debug, **kw)
+                        return_tuple = p.connector.sync(
+                            p,
+                            **filter_keywords(
+                                p.connector.sync,
+                                debug=debug,
+                                **kw
+                            )
+                        )
                     p._exists = None
                     if not isinstance(return_tuple, tuple):
                         return_tuple = (
@@ -237,13 +251,19 @@ def sync(
             ### Fetch the dataframe from the connector's `fetch()` method.
             try:
                 with Venv(get_connector_plugin(p.connector), debug=debug):
-                    df = p.fetch(debug=debug, **kw)
+                    df = p.fetch(
+                        **filter_keywords(
+                            p.fetch,
+                            debug=debug,
+                            **kw
+                        )
+                    )
 
             except Exception as e:
                 get_console().print_exception(
-                    suppress = [
-                        'meerschaum/core/Pipe/_sync.py', 
-                        'meerschaum/core/Pipe/_fetch.py', 
+                    suppress=[
+                        'meerschaum/core/Pipe/_sync.py',
+                        'meerschaum/core/Pipe/_fetch.py',
                     ]
                 )
                 msg = f"Failed to fetch data from {p.connector}:\n    {e}"
@@ -289,7 +309,7 @@ def sync(
             if not chunk_success:
                 return chunk_success, f"Unable to sync initial chunk for {p}:\n{chunk_msg}"
             if debug:
-                dprint(f"Successfully synced the first chunk, attemping the rest...")
+                dprint("Successfully synced the first chunk, attemping the rest...")
 
             failed_chunks = []
             def _process_chunk(_chunk):
@@ -309,7 +329,6 @@ def sync(
                     )
                 )
 
-
             results = sorted(
                 [(chunk_success, chunk_msg)] + (
                     list(pool.imap(_process_chunk, df))
@@ -329,7 +348,7 @@ def sync(
             retry_success = True
             if not success and any(success_bools):
                 if debug:
-                    dprint(f"Retrying failed chunks...")
+                    dprint("Retrying failed chunks...")
                 chunks_to_retry = [c for c in failed_chunks]
                 failed_chunks = []
                 for chunk in chunks_to_retry:
@@ -361,9 +380,9 @@ def sync(
         while run:
             with Venv(get_connector_plugin(self.instance_connector)):
                 return_tuple = p.instance_connector.sync_pipe(
-                    pipe = p,
-                    df = df,
-                    debug = debug,
+                    pipe=p,
+                    df=df,
+                    debug=debug,
                     **kw
                 )
             _retries += 1
@@ -382,7 +401,7 @@ def sync(
         _checkpoint(**kw)
         if self.cache_pipe is not None:
             if debug:
-                dprint(f"Caching retrieved dataframe.", **kw)
+                dprint("Caching retrieved dataframe.", **kw)
                 _sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
                 if not _sync_cache_tuple[0]:
                     warn(f"Failed to sync local cache for {self}.")
@@ -395,10 +414,10 @@ def sync(
         return _sync(self, df = df)
 
     from meerschaum.utils.threading import Thread
-    def default_callback(result_tuple : SuccessTuple):
+    def default_callback(result_tuple: SuccessTuple):
         dprint(f"Asynchronous result from {self}: {result_tuple}", **kw)
 
-    def default_error_callback(x : Exception):
+    def default_error_callback(x: Exception):
         dprint(f"Error received for {self}: {x}", **kw)
 
     if callback is None and debug:
@@ -407,12 +426,12 @@ def sync(
         error_callback = default_error_callback
     try:
         thread = Thread(
-            target = _sync,
-            args = (self,),
-            kwargs = {'df' : df},
-            daemon = False,
-            callback = callback,
-            error_callback = error_callback
+            target=_sync,
+            args=(self,),
+            kwargs={'df': df},
+            daemon=False,
+            callback=callback,
+            error_callback=error_callback,
         )
         thread.start()
     except Exception as e:
@@ -424,12 +443,13 @@ def sync(
 
 
 def get_sync_time(
-        self,
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        round_down: bool = False, 
-        debug: bool = False
-    ) -> Union['datetime', None]:
+    self,
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    apply_backtrack_interval: bool = False,
+    round_down: bool = False,
+    debug: bool = False
+) -> Union['datetime', None]:
     """
     Get the most recent datetime value for a Pipe.
 
@@ -443,6 +463,9 @@ def get_sync_time(
         If `True`, get the most recent datetime (honoring `params`).
         If `False`, get the oldest datetime (`ASC` instead of `DESC`).
 
+    apply_backtrack_interval: bool, default False
+        If `True`, subtract the backtrack interval from the sync time.
+
     round_down: bool, default False
         If `True`, round down the datetime value to the nearest minute.
 
@@ -461,15 +484,22 @@ def get_sync_time(
     with Venv(get_connector_plugin(self.instance_connector)):
         sync_time = self.instance_connector.get_sync_time(
             self,
-            params = params,
-            newest = newest,
-            debug = debug,
+            params=params,
+            newest=newest,
+            debug=debug,
         )
 
-    if not round_down or not isinstance(sync_time, datetime):
-        return sync_time
+    if round_down and isinstance(sync_time, datetime):
+        sync_time = round_time(sync_time, timedelta(minutes=1))
+
+    if apply_backtrack_interval and sync_time is not None:
+        backtrack_interval = self.get_backtrack_interval(debug=debug)
+        try:
+            sync_time -= backtrack_interval
+        except Exception as e:
+            warn(f"Failed to apply backtrack interval:\n{e}")
 
-    return round_time(sync_time, timedelta(minutes=1))
+    return sync_time
 
 
 def exists(

meerschaum/plugins/bootstrap.py

@@ -0,0 +1,333 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Define the bootstrapping wizard for creating plugins.
+"""
+
+import re
+import pathlib
+import meerschaum as mrsm
+from meerschaum.utils.typing import Any, SuccessTuple, Dict, List
+from meerschaum.utils.warnings import warn, info
+from meerschaum.utils.prompt import prompt, choose, yes_no
+from meerschaum.utils.formatting._shell import clear_screen
+
+FEATURE_CHOICES: Dict[str, str] = {
+    'fetch'    : 'Fetch data\n     (e.g. extracting from a remote API)\n',
+    'connector': 'Custom connector\n     (e.g. manage credentials)\n',
+    'action'   : 'New actions\n     (e.g. `mrsm sing song`)\n',
+    'api'      : 'New API endpoints\n     (e.g. `POST /my/new/endpoint`)\n',
+    'web'      : 'New web console page\n     (e.g. `/dash/my-web-app`)\n',
+}
+
+IMPORTS_LINES: Dict[str, str] = {
+    'stdlib': (
+        "from datetime import datetime, timedelta, timezone\n"
+    ),
+    'default': (
+        "import meerschaum as mrsm\n"
+        "from meerschaum.config import get_plugin_config, write_plugin_config\n"
+    ),
+    'connector': (
+        "from meerschaum.connectors import Connector, make_connector\n"
+    ),
+    'action': (
+        "from meerschaum.actions import make_action\n"
+    ),
+    'api': (
+        "from meerschaum.plugins import api_plugin\n"
+    ),
+    'web': (
+        "from meerschaum.plugins import web_page, dash_plugin\n"
+    ),
+    'api+web': (
+        "from meerschaum.plugins import api_plugin, web_page, dash_plugin\n"
+    ),
+}
+
+### TODO: Add feature for custom connectors.
+FEATURE_LINES: Dict[str, str] = {
+    'header': (
+        "#! /usr/bin/env python3\n"
+        "# -*- coding: utf-8 -*-\n\n"
+        "\"\"\"\n"
+        "Implement the plugin '{plugin_name}'.\n\n"
+        "See the Writing Plugins guide for more information:\n"
+        "https://meerschaum.io/reference/plugins/writing-plugins/\n"
+        "\"\"\"\n\n"
+    ),
+    'default': (
+        "__version__ = '0.0.1'\n"
+        "\n# Add any dependencies to `required` (similar to `requirements.txt`).\n"
+        "required: list[str] = []\n\n\n"
+    ),
+    'setup': (
+        "def setup(**kwargs) -> mrsm.SuccessTuple:\n"
+        "    \"\"\"Executed during installation and `mrsm setup plugin {plugin_name}`.\"\"\"\n"
+        "    return True, \"Success\"\n\n\n"
+    ),
+    'register': (
+        "def register(pipe: mrsm.Pipe):\n"
+        "    \"\"\"Return the default parameters for a new pipe.\"\"\"\n"
+        "    return {{\n"
+        "        'columns': {{\n"
+        "            'datetime': {dt_col_name},\n"
+        "        }}\n"
+        "    }}\n\n\n"
+    ),
+    'fetch': (
+        "def fetch(\n"
+        "    pipe: mrsm.Pipe,\n"
+        "    begin: datetime | None = None,\n"
+        "    end: datetime | None = None,\n"
+        "    **kwargs\n"
+        "):\n"
+        "    \"\"\"Return or yield dataframes.\"\"\"\n"
+        "    docs = []\n"
+        "    # populate docs with dictionaries (rows).\n"
+        "    return docs\n\n\n"
+    ),
+    'connector': (
+        "@make_connector\n"
+        "class {plugin_name_capitalized}Connector(Connector):\n"
+        "    \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
+        "    REQUIRED_ATTRIBUTES: list[str] = []\n"
+        "    \n"
+        "    def fetch(\n"
+        "        self,\n"
+        "        pipe: mrsm.Pipe,\n"
+        "        begin: datetime | None = None,\n"
+        "        end: datetime | None = None,\n"
+        "        **kwargs\n"
+        "    ):\n"
+        "        \"\"\"Return or yield dataframes.\"\"\"\n"
+        "        docs = []\n"
+        "        # populate docs with dictionaries (rows).\n"
+        "        return docs\n\n\n"
+    ),
+    'action': (
+        "@make_action\n"
+        "def {action_name}(**kwargs) -> mrsm.SuccessTuple:\n"
+        "    \"\"\"Run `mrsm {action_spaces}` to trigger.\"\"\"\n"
+        "    return True, \"Success\"\n\n\n"
+    ),
+    'api': (
+        "@api_plugin\n"
+        "def init_app(fastapi_app):\n"
+        "    \"\"\"Add new endpoints to the FastAPI app.\"\"\"\n\n"
+        "    import fastapi\n"
+        "    from meerschaum.api import manager\n\n"
+        "    @fastapi_app.get('/{plugin_name}')\n"
+        "    def get_my_endpoint(curr_user=fastapi.Depends(manager)):\n"
+        "        return {{'message': \"Hello from plugin '{plugin_name}'!\"}}\n\n\n"
+    ),
+    'web': (
+        "@dash_plugin\n"
+        "def init_dash(dash_app):\n"
+        "    \"\"\"Initialize the Plotly Dash application.\"\"\"\n\n"
+        "    import dash.html as html\n"
+        "    import dash.dcc as dcc\n"
+        "    from dash import Input, Output, State, no_update\n"
+        "    import dash_bootstrap_components as dbc\n\n"
+        "    # Create a new page at the path `/dash/{plugin_name}`.\n"
+        "    @web_page('{plugin_name}', login_required=False)\n"
+        "    def page_layout():\n"
+        "        \"\"\"Return the layout objects for this page.\"\"\"\n"
+        "        return dbc.Container([\n"
+        "            dcc.Location(id='{plugin_name}-location'),\n"
+        "            html.Div(id='output-div'),\n"
+        "        ])\n\n"
+        "    @dash_app.callback(\n"
+        "        Output('output-div', 'children'),\n"
+        "        Input('{plugin_name}-location', 'pathname'),\n"
+        "    )\n"
+        "    def render_page_on_url_change(pathname: str):\n"
+        "        \"\"\"Reload page contents when the URL path changes.\"\"\"\n"
+        "        return html.H1(\"Hello from plugin '{plugin_name}'!\")\n\n\n"
+    ),
+}
+
+def bootstrap_plugin(
+        plugin_name: str,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> SuccessTuple:
+    """
+    Prompt the user for features and create a plugin file.
+    """
+    from meerschaum.utils.misc import edit_file
+    plugins_dir_path = _get_plugins_dir_path()
+    clear_screen(debug=debug)
+    info(
+        "Answer the questions below to pick out features.\n"
+        + "    See the Writing Plugins guide for documentation:\n"
+        + "    https://meerschaum.io/reference/plugins/writing-plugins/ "
+        + "for documentation.\n"
+    )
+
+    plugin = mrsm.Plugin(plugin_name)
+    if plugin.is_installed():
+        uninstall_success, uninstall_msg = _ask_to_uninstall(plugin)
+        if not uninstall_success:
+            return uninstall_success, uninstall_msg
+        clear_screen(debug=debug)
+
+    features: List[str] = choose(
+        "Which of the following features would you like to add to your plugin?",
+        list(FEATURE_CHOICES.items()),
+        default = 'fetch',
+        multiple = True,
+        as_indices = True,
+        **kwargs
+    )
+
+    clear_screen(debug=debug)
+
+    action_name = ''
+    if 'action' in features:
+        action_name = _get_action_name()
+        clear_screen(debug=debug)
+    action_spaces = action_name.replace('_', ' ')
+
+    dt_col_name = None
+    if 'fetch' in features:
+        dt_col_name = _get_quoted_dt_col_name()
+        clear_screen(debug=debug)
+
+    plugin_labels = {
+        'plugin_name': plugin_name,
+        'plugin_name_capitalized': re.split(
+            r'[-_]', plugin_name.lower()
+        )[0].capitalize(),
+        'plugin_name_lower': plugin_name.lower(),
+        'action_name': action_name,
+        'action_spaces': action_spaces,
+        'dt_col_name': dt_col_name,
+    }
+
+    body_text = ""
+    body_text += FEATURE_LINES['header'].format(**plugin_labels)
+    body_text += IMPORTS_LINES['stdlib'].format(**plugin_labels)
+    body_text += IMPORTS_LINES['default'].format(**plugin_labels)
+    if 'connector' in features:
+        body_text += IMPORTS_LINES['connector'].format(**plugin_labels)
+    if 'action' in features:
+        body_text += IMPORTS_LINES['action'].format(**plugin_labels)
+    if 'api' in features and 'web' in features:
+        body_text += IMPORTS_LINES['api+web'].format(**plugin_labels)
+    elif 'api' in features:
+        body_text += IMPORTS_LINES['api'].format(**plugin_labels)
+    elif 'web' in features:
+        body_text += IMPORTS_LINES['web'].format(**plugin_labels)
+
+    body_text += "\n"
+    body_text += FEATURE_LINES['default'].format(**plugin_labels)
+    body_text += FEATURE_LINES['setup'].format(**plugin_labels)
+
+    if 'fetch' in features:
+        body_text += FEATURE_LINES['register'].format(**plugin_labels)
+        body_text += FEATURE_LINES['fetch'].format(**plugin_labels)
+
+    if 'connector' in features:
+        body_text += FEATURE_LINES['connector'].format(**plugin_labels)
+
+    if 'action' in features:
+        body_text += FEATURE_LINES['action'].format(**plugin_labels)
+
+    if 'api' in features:
+        body_text += FEATURE_LINES['api'].format(**plugin_labels)
+
+    if 'web' in features:
+        body_text += FEATURE_LINES['web'].format(**plugin_labels)
+
+    try:
+        plugin_path = plugins_dir_path / (plugin_name + '.py')
+        with open(plugin_path, 'w+', encoding='utf-8') as f:
+            f.write(body_text.rstrip())
+    except Exception as e:
+        error_msg = f"Failed to write file '{plugin_path}':\n{e}"
+        return False, error_msg
+
+    clear_screen(debug=debug)
+    mrsm.pprint((True, f"Successfully created file '{plugin_path}'."))
+    try:
+        _ = prompt(
+            f"Press [Enter] to edit plugin '{plugin_name}',"
+            + " [CTRL+C] to skip.",
+            icon = False,
+        )
+    except (KeyboardInterrupt, Exception):
+        return True, "Success"
+
+    edit_file(plugin_path, debug=debug)
+    return True, "Success"
+
+
+def _get_plugins_dir_path() -> pathlib.Path:
+    from meerschaum.config.paths import PLUGINS_DIR_PATHS
+    
+    if not PLUGINS_DIR_PATHS:
+        raise EnvironmentError("No plugin dir path could be found.")
+
+    if len(PLUGINS_DIR_PATHS) == 1:
+        return PLUGINS_DIR_PATHS[0]
+
+    return pathlib.Path(
+        choose(
+            "In which directory do you want to write your plugin?",
+            [path.as_posix() for path in PLUGINS_DIR_PATHS],
+            numeric = True,
+            multiple = False,
+            default = PLUGINS_DIR_PATHS[0].as_posix(),
+        )
+    )
+ 
+
+def _ask_to_uninstall(plugin: mrsm.Plugin, **kwargs: Any) -> SuccessTuple:
+    from meerschaum._internal.entry import entry
+    warn(f"Plugin '{plugin}' is already installed!", stack=False)
+    uninstall_plugin = yes_no(
+        f"Do you want to first uninstall '{plugin}'?",
+        default = 'n',
+        **kwargs
+    )
+    if not uninstall_plugin:
+        return False, f"Plugin '{plugin}' already exists."
+
+    return entry(['uninstall', 'plugin', plugin.name, '-f'])
+
+
+def _get_action_name() -> str:
+    while True:
+        try:
+            action_name = prompt(
+                "What is name of your action?\n    "
+                + "(separate subactions with spaces, e.g. `sing song`):"
+            ).replace(' ', '_')
+        except KeyboardInterrupt:
+            return False, "Aborted plugin creation."
+
+        if action_name:
+            break
+        warn("Please enter an action.", stack=False)
+    return action_name
+
+
+def _get_quoted_dt_col_name() -> str:
+    try:
+        dt_col_name = prompt(
+            "Enter the datetime column name ([CTRL+C] to skip):"
+        )
+    except (Exception, KeyboardInterrupt):
+        dt_col_name = None
+
+    if dt_col_name is None:
+        dt_col_name = 'None'
+    elif '"' in dt_col_name or "'" in dt_col_name:
+        dt_col_name = f"\"\"\"{dt_col_name}\"\"\""
+    else:
+        dt_col_name = f"\"{dt_col_name}\""
+
+    return dt_col_name

meerschaum/utils/formatting/__init__.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import platform
 import os
 import sys
-from meerschaum.utils.typing import Optional, Union, Any
+from meerschaum.utils.typing import Optional, Union, Any, Dict
 from meerschaum.utils.formatting._shell import make_header
 from meerschaum.utils.formatting._pprint import pprint
 from meerschaum.utils.formatting._pipes import (
@@ -298,6 +298,7 @@ def print_options(
         header: Optional[str] = None,
         num_cols: Optional[int] = None,
         adjust_cols: bool = True,
+        sort_options: bool = False,
         **kw
     ) -> None:
     """
@@ -339,6 +340,8 @@ def print_options(
     _options = []
     for o in options:
         _options.append(str(o))
+    if sort_options:
+        _options = sorted(_options)
     _header = f"Available {name}" if header is None else header
 
     if num_cols is None:
@@ -349,7 +352,7 @@ def print_options(
             print()
             print(make_header(_header))
         ### print actions
-        for option in sorted(_options):
+        for option in _options:
             if not nopretty:
                 print("  - ", end="")
             print(option)
@@ -385,7 +388,7 @@ def print_options(
 
     if _header is not None:
         table = Table(
-            title = '\n' + _header,
+            title = ('\n' + _header) if header else header,
             box = box.SIMPLE,
             show_header = False,
             show_footer = False,
@@ -397,13 +400,22 @@ def print_options(
     for i in range(num_cols):
         table.add_column()
 
-    chunks = iterate_chunks(
-        [Text.from_ansi(highlight_pipes(o)) for o in sorted(_options)],
-        num_cols,
-        fillvalue=''
-    )
-    for c in chunks:
-        table.add_row(*c)
+    if len(_options) < 12:
+        # If fewer than 12 items, use a single column
+        for option in _options:
+            table.add_row(Text.from_ansi(highlight_pipes(option)))
+    else:
+        # Otherwise, use multiple columns as before
+        num_rows = (len(_options) + num_cols - 1) // num_cols
+        for i in range(num_rows):
+            row = []
+            for j in range(num_cols):
+                index = i + j * num_rows
+                if index < len(_options):
+                    row.append(Text.from_ansi(highlight_pipes(_options[index])))
+                else:
+                    row.append('')
+            table.add_row(*row)
 
     get_console().print(table)
     return None