meerschaum 2.1.0rc2__py3-none-any.whl → 2.1.1rc1__py3-none-any.whl

meerschaum/actions/bootstrap.py

@@ -15,7 +15,7 @@ def bootstrap(
         **kw: Any
     ) -> SuccessTuple:
     """
-    Bootstrap an element (pipes, connectors, config).
+    Launch an interactive wizard to bootstrap pipes or connectors.
     
     Example:
         `bootstrap pipes`
@@ -24,7 +24,6 @@ def bootstrap(
     from meerschaum.actions import choose_subaction
     options = {
         'pipes'      : _bootstrap_pipes,
-        'config'     : _bootstrap_config,
         'connectors' : _bootstrap_connectors,
     }
     return choose_subaction(action, options, **kw)

meerschaum/actions/delete.py

@@ -64,6 +64,7 @@ def _complete_delete(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['delete'] + action), **kw)
 
+
 def _delete_pipes(
         debug: bool = False,
         yes: bool = False,
@@ -79,13 +80,26 @@ def _delete_pipes(
     from meerschaum.utils.prompt import yes_no
     from meerschaum.utils.formatting import pprint, highlight_pipes
     from meerschaum.utils.warnings import warn
+    from meerschaum.actions import actions
     pipes = get_pipes(as_list=True, debug=debug, **kw)
     if len(pipes) == 0:
         return False, "No pipes to delete."
+
+    _ = kw.pop('action', None)
+    _ = actions['drop'](
+        ['pipes'],
+        yes = yes,
+        force = force,
+        noask = noask,
+        debug = debug,
+        **kw
+    )
+
     question = "Are you sure you want to delete these pipes? This can't be undone!\n"
     for p in pipes:
         question += f"    - {p}" + "\n"
     question = highlight_pipes(question)
+
     answer = force
     if force:
         answer = True
@@ -98,7 +112,7 @@ def _delete_pipes(
     success_dict = {}
 
     for p in pipes:
-        success_tuple = p.delete(debug=debug)
+        success_tuple = p.delete(drop=False, debug=debug)
         success_dict[p] = success_tuple[1]
         if success_tuple[0]:
             successes += 1

meerschaum/actions/sync.py

@@ -164,7 +164,7 @@ def _pipes_lap(
                     remaining_count -= 1
             pipes_queue.task_done()
 
-    sync_function_source = dill.source.getsource(_wrap_sync_pipe)
+    sync_function_source = dill.source.getsource(_wrap_pipe)
     fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
 
     def sync_pipe(p):
@@ -173,7 +173,7 @@ def _pipes_lap(
         """
         ### If no timeout is specified, handle syncing in the current thread.
         if timeout_seconds is None:
-            return _wrap_sync_pipe(p, **all_kw)
+            return _wrap_pipe(p, **all_kw)
         _success_tuple = False, "Nothing returned."
         def write_line(line):
             nonlocal _success_tuple
@@ -209,7 +209,7 @@ def _pipes_lap(
             + f"""print(
                     '{fence_begin}'
                     + json.dumps(
-                        _wrap_sync_pipe(
+                        _wrap_pipe(
                             pipe,
                             **json.loads({json.dumps(json.dumps(all_kw))})
                         )
@@ -384,7 +384,7 @@ def _sync_pipes(
     return (len(success_pipes) > 0 if success_pipes is not None else False), msg
 
 
-def _wrap_sync_pipe(
+def _wrap_pipe(
         pipe,
         unblock: bool = False,
         force: bool = False,

meerschaum/api/routes/_pipes.py

@@ -87,7 +87,7 @@ def delete_pipe(
         ),
     ):
     """
-    Delete a Pipe, dropping its table.
+    Delete a Pipe (without dropping its table).
     """
     from meerschaum.config import get_config
     allow_actions = get_config('system', 'api', 'permissions', 'actions', 'non_admin')
@@ -99,19 +99,19 @@ def delete_pipe(
             " Under the keys `api:permissions:actions`, " +
             "you can toggle non-admin actions."
         )
-    pipe_object = get_pipe(connector_keys, metric_key, location_key)
-    if not is_pipe_registered(pipe_object, pipes(refresh=True)):
+    pipe = get_pipe(connector_keys, metric_key, location_key)
+    if not is_pipe_registered(pipe, pipes(refresh=True)):
         raise fastapi.HTTPException(
-            status_code=409, detail=f"{pipe_object} is not registered."
+            status_code=409, detail=f"{pipe} is not registered."
         )
-    results = get_api_connector().delete_pipe(pipe_object, debug=debug)
+    results = get_api_connector().delete_pipe(pipe, debug=debug)
     pipes(refresh=True)
 
     return results
 
 
 @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
-def delete_pipe(
+def drop_pipe(
         connector_keys: str,
         metric_key: str,
         location_key: str,
@@ -120,7 +120,7 @@ def delete_pipe(
         ),
     ):
     """
-    Drropping a pipes' table (without deleting its registration).
+    Dropping a pipes' table (without deleting its registration).
     """
     from meerschaum.config import get_config
     allow_actions = get_config('system', 'api', 'permissions', 'actions', 'non_admin')
@@ -133,10 +133,6 @@ def delete_pipe(
             "you can toggle non-admin actions."
         )
     pipe_object = get_pipe(connector_keys, metric_key, location_key)
-    if not is_pipe_registered(pipe_object, pipes(refresh=True)):
-        raise fastapi.HTTPException(
-            status_code=409, detail=f"{pipe_object} is not registered."
-        )
     results = get_api_connector().drop_pipe(pipe_object, debug=debug)
     pipes(refresh=True)
     return results

meerschaum/config/__init__.py

@@ -24,8 +24,6 @@ from meerschaum.config._paths import (
     DEFAULT_CONFIG_DIR_PATH,
 )
 from meerschaum.config._patch import (
-    #  permanent_patch_config,
-    #  patch_config,
     apply_patch_to_config,
 )
 __all__ = ('get_plugin_config', 'write_plugin_config', 'get_config', 'write_config', 'set_config',)

meerschaum/config/_default.py

@@ -59,6 +59,9 @@ default_system_config = {
             'pandas': 'pandas',
         },
         'sql': {
+            'instance': {
+                'stale_temporary_tables_minutes': 1440,
+            },
             'chunksize': 100_000,
             'poolclass': 'sqlalchemy.pool.QueuePool',
             'create_engine': {

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.0rc2"
+__version__ = "2.1.1rc1"

meerschaum/config/static/__init__.py

@@ -40,6 +40,10 @@ STATIC_CONFIG: Dict[str, Any] = {
         },
         'webterm_job_name': '_webterm',
     },
+    'sql': {
+        'internal_schema': '_mrsm_internal',
+        'instance_schema': 'mrsm',
+    },
     'environment': {
         'config': 'MRSM_CONFIG',
         'patch': 'MRSM_PATCH',

meerschaum/connectors/sql/SQLConnector.py

@@ -7,6 +7,7 @@ Interface with SQL servers using sqlalchemy.
 """
 
 from __future__ import annotations
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, Union
 
 from meerschaum.connectors import Connector
@@ -28,7 +29,7 @@ class SQLConnector(Connector):
     from ._sql import read, value, exec, execute, to_sql, exec_queries
     from meerschaum.utils.sql import test_connection
     from ._fetch import fetch, get_pipe_metadef
-    from ._cli import cli
+    from ._cli import cli, _cli_exit
     from ._pipes import (
         fetch_pipes_keys,
         create_indices,
@@ -78,6 +79,13 @@ class SQLConnector(Connector):
         get_user_attributes,
     )
     from ._uri import from_uri, parse_uri
+    from ._instance import (
+        _get_temporary_tables_pipe,
+        _log_temporary_tables_creation,
+        _drop_temporary_table,
+        _drop_temporary_tables,
+        _drop_old_temporary_tables,
+    )
     
     def __init__(
         self,
@@ -259,14 +267,46 @@ class SQLConnector(Connector):
             return ':memory:' not in self.URI
         return True
 
+
     @property
     def metadata(self):
+        """
+        Return the metadata bound to this configured schema.
+        """
         from meerschaum.utils.packages import attempt_import
         sqlalchemy = attempt_import('sqlalchemy')
         if '_metadata' not in self.__dict__:
             self._metadata = sqlalchemy.MetaData(schema=self.schema)
         return self._metadata
 
+
+    @property
+    def instance_schema(self):
+        """
+        Return the schema name for Meerschaum tables. 
+        """
+        return self.schema
+
+
+    @property
+    def internal_schema(self):
+        """
+        Return the schema name for internal tables. 
+        """
+        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum.utils.packages import attempt_import
+        from meerschaum.utils.sql import NO_SCHEMA_FLAVORS
+        schema_name = self.__dict__.get('internal_schema', None) or (
+            STATIC_CONFIG['sql']['internal_schema']
+            if self.flavor not in NO_SCHEMA_FLAVORS
+            else self.schema
+        )
+
+        if '_internal_schema' not in self.__dict__:
+            self._internal_schema = schema_name
+        return self._internal_schema
+
+
     @property
     def db(self) -> Optional[databases.Database]:
         from meerschaum.utils.packages import attempt_import
@@ -307,8 +347,8 @@ class SQLConnector(Connector):
         if 'schema' in self.__dict__:
             return self.__dict__['schema']
 
-        uri = self.__dict__.get('uri', self.URI)
-        _schema = self.parse_uri(uri).get('schema', None)
+        sqlalchemy = mrsm.attempt_import('sqlalchemy')
+        _schema = sqlalchemy.inspect(self.engine).default_schema_name
         self.__dict__['schema'] = _schema
         return _schema
 

meerschaum/connectors/sql/_cli.py

@@ -7,6 +7,9 @@ Launch into a CLI environment to interact with the SQL Connector
 """
 
 from __future__ import annotations
+import os
+import json
+import copy
 ### NOTE: This import adds `Iterable` to collections, which is needed by some CLIs.
 from meerschaum.utils.typing import SuccessTuple
 
@@ -27,9 +30,32 @@ cli_deps = {
     'mycli': ['cryptography'],
 }
 
+
 def cli(
         self,
-        debug : bool = False
+        debug: bool = False,
+    ) -> SuccessTuple:
+    """
+    Launch a subprocess for an interactive CLI.
+    """
+    from meerschaum.utils.venv import venv_exec
+    env = copy.deepcopy(dict(os.environ))
+    env[f'MRSM_SQL_{self.label.upper()}'] = json.dumps(self.meta)
+    cli_code = (
+        "import meerschaum as mrsm\n"
+        f"conn = mrsm.get_connector('sql:{self.label}')\n"
+        f"conn._cli_exit()"
+    )
+    try:
+        _ = venv_exec(cli_code, venv=None, debug=debug, capture_output=False)
+    except Exception as e:
+        return False, f"[{self}] Failed to start CLI:\n{e}"
+    return True, "Success"
+
+
+def _cli_exit(
+        self,
+        debug: bool = False
     ) -> SuccessTuple:
     """Launch an interactive CLI for the SQLConnector's flavor."""
     from meerschaum.utils.packages import venv_exec, attempt_import
@@ -85,8 +111,6 @@ def cli(
             + "finally:\n"
             + "    ms_object.shutdown()"
         )
-    elif self.flavor == 'duckdb':
-        launch_cli = ()
 
     try:
         if debug:

meerschaum/connectors/sql/_instance.py

@@ -0,0 +1,164 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Define utilities for instance connectors.
+"""
+
+import time
+from datetime import datetime, timezone, timedelta
+import meerschaum as mrsm
+from meerschaum.utils.typing import Dict, SuccessTuple, Optional, Union, List
+from meerschaum.utils.warnings import warn
+
+def _get_temporary_tables_pipe(self) -> mrsm.Pipe:
+    """
+    Return a pipe for logging temporary tables.
+    """
+    return mrsm.Pipe(
+        "temporary", "tables",
+        target = "mrsm_temp_tables",
+        temporary = True,
+        instance = self,
+        columns = {
+            'datetime': 'date_created',
+            'table': 'table',
+            'ready_to_drop': 'ready_to_drop',
+        },
+        dtypes = {
+            'ready_to_drop': 'datetime',
+        },
+        parameters = {
+            'schema': self.instance_schema,
+        },
+    )
+
+
+def _log_temporary_tables_creation(
+        self,
+        tables: Union[str, List[str]],
+        ready_to_drop: bool = False,
+        debug: bool = False,
+    ) -> SuccessTuple:
+    """
+    Log a temporary table's creation for later deletion.
+    """
+    if isinstance(tables, str):
+        tables = [tables]
+    temporary_tables_pipe = self._get_temporary_tables_pipe()
+    now = datetime.now(timezone.utc).replace(tzinfo=None)
+    return temporary_tables_pipe.sync(
+        [
+            {
+                'date_created': now,
+                'table': table,
+                'ready_to_drop': (now if ready_to_drop else None),
+            }
+            for table in tables
+        ],
+        check_existing = False,
+        debug = debug,
+    )
+
+
+def _drop_temporary_table(
+        self,
+        table: str,
+        debug: bool = False,
+    ) -> SuccessTuple:
+    """
+    Drop a temporary table and clear it from the internal table.
+    """
+    from meerschaum.utils.sql import sql_item_name, table_exists
+    temporary_tables_pipe = self._get_temporary_tables_pipe()
+    if table_exists(table, self, self.internal_schema, debug=debug):
+        return True, "Success"
+    drop_query = f"DROP TABLE IF EXISTS " + sql_item_name(
+        table, self.flavor, schema=self.internal_schema
+    )
+    drop_success = self.exec(drop_query, silent=True, debug=debug) is not None
+    drop_msg = "Success" if drop_success else f"Failed to drop temporary table '{table}'."
+    return drop_success, drop_msg
+
+
+def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
+    """
+    Drop all tables in the internal schema that are marked as ready to be dropped.
+    """
+    from meerschaum.utils.sql import sql_item_name, table_exists
+    from meerschaum.utils.misc import items_str
+
+    temporary_tables_pipe = self._get_temporary_tables_pipe()
+    df = temporary_tables_pipe.get_data(['table'], params={'ready_to_drop': '_None'}, debug=debug)
+    if df is None:
+        return True, "Success"
+
+    dropped_tables = []
+    failed_tables = []
+    for table in df['table']:
+        drop_success, drop_msg = self._drop_temporary_table(table, debug=debug)
+        if not drop_success:
+            failed_tables.append(table)
+            continue
+        dropped_tables.append(table)
+
+    if dropped_tables:
+        temporary_tables_pipe.clear(params={'table': dropped_tables}, debug=debug)
+
+    success = len(failed_tables) == 0
+    msg = (
+        "Success"
+        if success
+        else (
+            "Failed to drop stale temporary tables "
+            + f"{items_str(failed_tables)}."
+        )
+    )
+    return success, msg
+
+
+def _drop_old_temporary_tables(
+        self,
+        refresh: bool = True,
+        debug: bool = False,
+    ) -> SuccessTuple:
+    """
+    Drop temporary tables older than the configured interval (24 hours by default).
+    """
+    from meerschaum.config import get_config
+    last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
+    now_ts = time.perf_counter()
+    if refresh or not last_check or (now_ts - last_check) > 60:
+        self._stale_temporary_tables_check_timestamp = now_ts
+        return self._drop_temporary_tables(debug=debug)
+
+    stale_temporary_tables_minutes = get_config(
+        'system', 'connectors', 'sql', 'instance', 'stale_temporary_tables_minutes'
+    )
+    temporary_tables_pipe = self._get_temporary_tables_pipe()
+    now = datetime.now(timezone.utc).replace(tzinfo=None)
+    end = now - timedelta(minutes=stale_temporary_tables_minutes)
+    df = temporary_tables_pipe.get_data(end=end, debug=debug)
+    if df is None:
+        return True, "Success"
+
+    ### Insert new records with the current time (skipping updates to avoid recursion).
+    docs = [
+        {
+            'date_created': now,
+            'table': doc['table'],
+            'ready_to_drop': now,
+        }
+        for doc in df.to_dict(orient='records')
+    ]
+    if docs:
+        update_temporary_success, update_temporary_msg = temporary_tables_pipe.sync(
+            docs,
+            check_existing = False,
+            debug = debug,
+        )
+        if not update_temporary_success:
+            warn(update_temporary_msg)
+
+    return self._drop_temporary_tables(debug=debug)