meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/_internal/static.py

@@ -0,0 +1,359 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define internal static config (formerly `meerschaum.config.static`).
+"""
+
+import os
+import random
+import string
+from typing import Dict, Any
+
+__all__ = ('STATIC_CONFIG',)
+
+_default_create_engine_args = {
+    #  'method': 'multi',
+    'pool_size': (os.cpu_count() or 5),
+    'max_overflow': (os.cpu_count() or 10),
+    'pool_recycle': 3600,
+    'connect_args': {},
+}
+_default_db_requirements = {
+    'username',
+    'password',
+    'host',
+    'database',
+}
+SERVER_ID: str = os.environ.get('MRSM_SERVER_ID', ''.join(random.sample(string.ascii_lowercase+string.digits, 6)))
+STATIC_CONFIG: Dict[str, Any] = {
+    'api': {
+        'endpoints': {
+            'index': '/',
+            'favicon': '/favicon.ico',
+            'plugins': '/plugins',
+            'pipes': '/pipes',
+            'metadata': '/metadata',
+            'actions': '/actions',
+            'jobs': '/jobs',
+            'logs': '/logs',
+            'users': '/users',
+            'tokens': '/tokens',
+            'login': '/login',
+            'connectors': '/connectors',
+            'version': '/version',
+            'chaining': '/chaining',
+            'websocket': '/ws',
+            'dash': '/dash',
+            'webterm': r'/webterm/{session_id}',
+            'webterm_websocket': r'/websocket/{session_id}',
+            'info': '/info',
+            'healthcheck': '/healthcheck',
+            'docs': '/docs',
+            'redoc': '/redoc',
+            'openapi': '/openapi.json',
+        },
+        'oauth': {
+            'token_expires_minutes': 720,
+        },
+        'webterm_job_name': '_webterm',
+        'default_timeout': 600,
+        'jobs': {
+            'stdin_message': 'MRSM_STDIN',
+            'stop_message': 'MRSM_STOP',
+            'metadata_cache_seconds': 5,
+            'temp_prefix': '.api-temp-',
+        },
+    },
+    'sql': {
+        'internal_schema': '_mrsm_internal',
+        'instance_schema': 'mrsm',
+        'default_create_engine_args': _default_create_engine_args,
+        'create_engine_flavors': {
+            'timescaledb': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'timescaledb-ha': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'postgresql': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'postgis': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'citus': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'mssql': {
+                'engine': 'mssql+pyodbc',
+                'create_engine': {
+                    'fast_executemany': True,
+                    'use_insertmanyvalues': False,
+                    'isolation_level': 'AUTOCOMMIT',
+                    'use_setinputsizes': False,
+                    'pool_pre_ping': True,
+                    'ignore_no_transaction_on_rollback': True,
+                },
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': None,
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 1433,
+                    'options': (
+                        "driver=ODBC Driver 18 for SQL Server"
+                        "&UseFMTONLY=Yes"
+                        "&TrustServerCertificate=yes"
+                        "&Encrypt=no"
+                        "&MARS_Connection=yes"
+                    ),
+                },
+            },
+            'mysql': {
+                'engine': 'mysql+pymysql',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 3306,
+                },
+            },
+            'mariadb': {
+                'engine': 'mysql+pymysql',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 3306,
+                },
+            },
+            'oracle': {
+                'engine': 'oracle+oracledb',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': None,
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 1521,
+                },
+            },
+            'sqlite': {
+                'engine': 'sqlite',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': {'database'},
+                'defaults': {},
+            },
+            'duckdb': {
+                'engine': 'duckdb',
+                'create_engine': {},
+                'omit_create_engine': {'ALL',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': '',
+                'defaults': {},
+            },
+            'cockroachdb': {
+                'engine': 'cockroachdb',
+                'omit_create_engine': {'method',},
+                'create_engine': _default_create_engine_args,
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': {'host'},
+                'defaults': {
+                    'port': 26257,
+                    'database': 'defaultdb',
+                    'username': 'root',
+                    'password': 'admin',
+                },
+            },
+        },
+    },
+    'valkey': {
+        'colon': '-_',
+    },
+    'environment': {
+        'config': 'MRSM_CONFIG',
+        'config_dir': 'MRSM_CONFIG_DIR',
+        'patch': 'MRSM_PATCH',
+        'root': 'MRSM_ROOT_DIR',
+        'plugins': 'MRSM_PLUGINS_DIR',
+        'venvs': 'MRSM_VENVS_DIR',
+        'runtime': 'MRSM_RUNTIME',
+        'work_dir': 'MRSM_WORK_DIR',
+        'user': 'MRSM_USER',
+        'dep_group': 'MRSM_DEP_GROUP',
+        'home': 'MRSM_HOME',
+        'src': 'MRSM_SRC',
+        'uid': 'MRSM_UID',
+        'gid': 'MRSM_GID',
+        'noask': 'MRSM_NOASK',
+        'noninteractive': 'MRSM_NONINTERACTIVE',
+        'id': 'MRSM_SERVER_ID',
+        'test_flavors': 'MRSM_TEST_FLAVORS',
+        'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
+        'systemd_delete_job': 'MRSM_SYSTEMD_DELETE_JOB',
+        'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
+        'prefix': 'MRSM_',
+    },
+    'config': {
+        'default_filetype': 'json',
+        'symlinks_key': '_symlinks',
+    },
+    'system': {
+        'arguments': {
+            'sub_decorators': (
+                '[',
+                ']'
+            ),
+            'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
+            'failure_key': '_argparse_exception',
+            'and_key': '+',
+            'escaped_and_key': '++',
+            'pipeline_key': ':',
+            'escaped_pipeline_key': '::',
+        },
+        'urls': {
+            'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',
+        },
+        'success': {
+            'ignore': (
+                'Success',
+                'success'
+                'Succeeded',
+                '',
+                None,
+            ),
+        },
+        'prompt': {
+            'web': False,
+        },
+        'fetch_pipes_keys': {
+            'negation_prefix': '_',
+        },
+    },
+    'connectors': {
+        'default_label': 'main',
+    },
+    'dtypes': {
+        'datetime': {
+            'default_precision_unit': 'microsecond',
+        },
+    },
+    'stack': {
+        'dollar_standin': '<DOLLAR>', ### Temporary replacement for docker-compose.yaml.
+    },
+    'users': {
+        'password_hash': {
+            'algorithm_name': 'sha256',
+            'salt_bytes': 16,
+            'schemes': [
+                'pbkdf2_sha256',
+            ],
+            'default': 'pbkdf2_sha256',
+            'pbkdf2_sha256__default_rounds': 1_000_000,
+        },
+        'min_username_length': 1,
+        'max_username_length': 60,
+        'min_password_length': 5,
+    },
+    'plugins': {
+        'repo_separator': '@',
+        'lock_sleep_total': 1.0,
+        'lock_sleep_increment': 0.1,
+    },
+    'pipes': {
+        'dtypes': {
+            'min_ratio_columns_changed_for_full_astype': 0.5,
+        },
+        'max_bound_time_days': 36525,
+    },
+    'jobs': {
+        'check_restart_seconds': 1.0,
+        'stop_token': '<------- MRSM_STOP_TOKEN ------->',
+        'clear_token': '<------- MRSM_CLEAR_TOKEN ------->',
+        'flush_token': '<------- MRSM_FLUSH_TOKEN ------->\n',
+    },
+    'tokens': {
+        'minimum_length': 24,
+        'maximum_length': 32,
+        'hash_rounds': 100_000,
+        'scopes': {
+            'pipes:read': "Read pipes' parameters and the contents of target tables.",
+            'pipes:write': "Update pipes' parameters and sync to target tables.",
+            'pipes:drop': "Drop target tables.",
+            'pipes:delete': "Delete pipes' parameters and drop target tables.",
+            'actions:execute': "Execute arbitrary actions.",
+            'connectors:read': "Read the available connectors.",
+            'jobs:read': "Read jobs' properties",
+            'jobs:write': "Write jobs' properties",
+            'jobs:execute': "Run jobs.",
+            'jobs:delete': "Delete jobs.",
+            'logs:read': "Read jobs' logs.",
+            'jobs:stop': "Stop running jobs.",
+            'jobs:pause': "Pause running jobs.",
+            'instance:read': "Read an instance's system-level metadata.",
+            'instance:chain': "Allow chaining API instances using the associated credentials.",
+            'plugins:write': "Register and update plugins' metadata.",
+            'plugins:read': "Read attributes of registered plugins.",
+            'plugins:delete': "Delete plugins (owned by user) from the repository.",
+            'users:read': "Read metadata about the associated account.",
+            'users:write': "Write metadata for the associated account.",
+            'users:register': "Register new user accounts.",
+            'users:delete': "Delete the associated user account (or other users for admins).",
+        },
+    },
+}

meerschaum/_internal/term/TermPageHandler.py

@@ -10,11 +10,10 @@ from __future__ import annotations
 from typing import Any
 
 import meerschaum as mrsm
-from meerschaum.utils.warnings import warn
 tornado_web = mrsm.attempt_import('tornado.web', lazy=False)
 terminado = mrsm.attempt_import('terminado', lazy=False)
 
-tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
 
 
 class TermPageHandler(tornado_web.RequestHandler):

meerschaum/_internal/term/__init__.py

@@ -8,6 +8,9 @@ Build the web console virtual terminal using Tornado and xterm.js.
 
 from __future__ import annotations
 
+import os
+import json
+import pathlib
 from typing import Optional, Tuple
 
 import meerschaum as mrsm
@@ -16,7 +19,6 @@ from meerschaum._internal.term.TermPageHandler import TermPageHandler, CustomTer
 from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
 from meerschaum.utils.venv import venv_executable
 from meerschaum.utils.misc import is_tmux_available
-from meerschaum.utils.daemon._names import get_new_daemon_name
 
 tornado, tornado_ioloop, terminado = attempt_import(
     'tornado', 'tornado.ioloop', 'terminado', lazy=False,
@@ -25,6 +27,8 @@ tornado, tornado_ioloop, terminado = attempt_import(
 
 def get_webterm_app_and_manager(
     instance_keys: Optional[str] = None,
+    port: Optional[int] = None,
+    env_path: Optional[pathlib.Path] = None,
 ) -> Tuple[
     tornado.web.Application,
     terminado.UniqueTermManager,
@@ -36,19 +40,49 @@ def get_webterm_app_and_manager(
     -------
     A tuple of the Tornado web application and term manager.
     """
+    from meerschaum.config.environment import get_env_vars
+    from meerschaum._internal.static import STATIC_CONFIG
+    if env_path is None:
+        from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
+        env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
+
+    env_vars = get_env_vars()
+    env_dict = {
+        env_var: os.environ[env_var]
+        for env_var in env_vars
+        if env_var not in (
+            STATIC_CONFIG['environment']['systemd_log_path'],
+            STATIC_CONFIG['environment']['systemd_result_path'],
+            STATIC_CONFIG['environment']['systemd_delete_job'],
+            STATIC_CONFIG['environment']['systemd_stdin_path'],
+            STATIC_CONFIG['environment']['daemon_id'],
+        )
+    }
+    with open(env_path, 'w+', encoding='utf-8') as f:
+        json.dump(env_dict, f)
+
     shell_kwargs_str = f"mrsm_instance='{instance_keys}'" if instance_keys else ""
     commands = [
         venv_executable(None),
         '-c',
-        "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
-        "from meerschaum._internal.entry import get_shell; "
+        "import os\n"
+        "import pathlib\n"
+        "import json\n"
+        f"env_path = pathlib.Path('{env_path.as_posix()}')\n"
+        "with open(env_path, 'r', encoding='utf-8') as f:\n"
+        "    env_dict = json.load(f)\n"
+        "os.environ.update(env_dict)\n"
+        "_ = os.environ.pop('COLUMNS', None)\n"
+        "_ = os.environ.pop('LINES', None)\n"
+        "from meerschaum._internal.entry import get_shell\n"
         f"get_shell({shell_kwargs_str}).cmdloop()"
     ]
-    webterm_cf = mrsm.get_config('system', 'webterm')
+    webterm_cf = mrsm.get_config('api', 'webterm')
     if webterm_cf.get('tmux', {}).get('enabled', False) and is_tmux_available():
-        commands = ['tmux', 'new-session', '-A', '-s', 'MRSM_SESSION'] + commands
+        commands = ['tmux', 'new-session', '-A', '-s', f'MRSM_SESSION--{port}'] + commands
 
-    term_manager = terminado.NamedTermManager(shell_command=commands)
+    term_manager = terminado.NamedTermManager(shell_command=commands, extra_env=env_dict)
+    term_manager._port = port
     handlers = [
         (
             r"/websocket/(.+)/?",

meerschaum/_internal/term/tools.py

@@ -6,13 +6,14 @@
 Utility functions regarding the webterm.
 """
 
+from typing import List, Optional
+
 import meerschaum as mrsm
-from meerschaum.utils.typing import List
 
 
 def is_webterm_running(
-    host: str,
-    port: int,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
     protocol: str = 'http',
     session_id: str = 'mrsm',
 ) -> int:
@@ -20,10 +21,12 @@ def is_webterm_running(
     Determine whether the webterm service is running on a given host and port.
     """
     requests = mrsm.attempt_import('requests', lazy=False)
+    host = host or mrsm.get_config('api', 'webterm', 'host')
+    port = port or mrsm.get_config('api', 'webterm', 'port')
     url = f'{protocol}://{host}:{port}/webterm/{session_id}'
     try:
         r = requests.get(url, timeout=3)
-    except Exception as e:
+    except Exception:
         return False
     if not r:
         return False
@@ -39,22 +42,44 @@ def kill_tmux_session(session: str) -> bool:
     return run_process(command, capture_output=True) == 0
 
 
-def get_mrsm_tmux_sessions() -> List[str]:
+def get_mrsm_tmux_sessions(port: Optional[int] = None) -> List[str]:
     """
     Return a list of tmux sessions created by Meerschaum.
     """
     from meerschaum.utils.process import run_process
-    tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+    tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
     command = ['tmux', 'ls']
     proc = run_process(command, capture_output=True, as_proc=True)
     if proc.returncode != 0:
         return []
+
+    port = port or mrsm.get_config('api', 'webterm', 'port')
+
     sessions = [
         line.split(':', maxsplit=1)[0]
         for line in proc.stdout.read().decode('utf-8').split('\n')
     ]
+    mrsm_sessions_ports = []
+    for session in sessions:
+        if '--' not in session:
+            continue
+
+        parts = session.split('--', maxsplit=1)
+        if len(parts) != 2:
+            continue
+
+        if not parts[0].endswith(tmux_suffix):
+            continue
+
+        try:
+            session_port = int(parts[1])
+        except Exception:
+            continue
+
+        mrsm_sessions_ports.append((session, session_port))
+
     return [
         session
-        for session in sessions
-        if session.endswith(tmux_suffix)
+        for session, session_port in mrsm_sessions_ports
+        if session_port == port
     ]

meerschaum/actions/__init__.py

@@ -9,7 +9,6 @@ Default actions available to the mrsm CLI.
 from __future__ import annotations
 from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict, SuccessTuple
 from meerschaum.utils.packages import get_modules_from_package
-_custom_actions = []
 
 __all__ = (
     'get_action',
@@ -307,6 +306,8 @@ __all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'g
 ### functions that do not begin with '_' from all submodules.
 from inspect import getmembers, isfunction
 actions = {}
+_custom_actions_plugins: Dict[str, str] = {}
+_plugins_actions: Dict[str, List[str]] = {}
 
 for module in modules:
     ### A couple important things happening here:
@@ -329,8 +330,8 @@ for module in modules:
                         if isfunction(ob[1])
                             ### check that the function belongs to the module
                             and ob[0] == module.__name__.replace('_', '').split('.')[-1]
-                            ### skip functions that start with '_'
-                            and ob[0][0] != '_'
+                            ### skip functions that start with '__'
+                            and ob[0][0] != '__'
             ]
         )
     )
@@ -362,4 +363,5 @@ __pdoc__ = {
 }
 for a in actions:
     __pdoc__[a] = False
-meerschaum.plugins.load_plugins()
+
+meerschaum.plugins.load_plugins(skip_if_loaded=True)