meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/__init__.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Copyright 2023 Bennett Meares
+Copyright 2025 Bennett Meares
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,13 +19,14 @@ limitations under the License.
 """
 
 import atexit
+
 from meerschaum.utils.typing import SuccessTuple
 from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
 from meerschaum.jobs import Job, make_executor
-from meerschaum.connectors import get_connector, Connector, make_connector
+from meerschaum.connectors import get_connector, Connector, InstanceConnector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
@@ -42,6 +43,7 @@ __all__ = (
     "get_config",
     "Pipe",
     "Plugin",
+    "SuccessTuple",
     "Venv",
     "Plugin",
     "Job",
@@ -55,6 +57,7 @@ __all__ = (
     "utils",
     "SuccessTuple",
     "Connector",
+    "InstanceConnector",
     "make_connector",
     "entry",
 )

meerschaum/_internal/__init__.py

@@ -6,3 +6,4 @@
 Modules, classes, and variables declared in _internal are not intended for general public use.
 """
 
+from meerschaum._internal.static import STATIC_CONFIG

meerschaum/_internal/arguments/_parse_arguments.py

@@ -23,7 +23,7 @@ def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
     """
     Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
     if pipeline_key not in sysargs:
         return sysargs, []
@@ -40,7 +40,7 @@ def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
     Split a `sysargs` list containing "and" keys (`+`)
     into a list of individual `sysargs`.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     and_key = STATIC_CONFIG['system']['arguments']['and_key']
 
     if not sysargs or and_key not in sysargs:
@@ -90,7 +90,7 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 
     """
     import shlex
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum._internal.arguments._parser import parser
 
     global _loaded_plugins_args
@@ -360,7 +360,7 @@ def remove_leading_action(
     """
     from meerschaum.actions import get_action, get_main_action_name
     from meerschaum.utils.warnings import warn
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     action_function = get_action(action, _actions=_actions)
     if action_function is None:
         return action

meerschaum/_internal/arguments/_parser.py

@@ -195,6 +195,7 @@ groups['pipes'] = parser.add_argument_group(title='Pipes options')
 groups['sync'] = parser.add_argument_group(title='Sync options')
 groups['api'] = parser.add_argument_group(title='API options')
 groups['plugins'] = parser.add_argument_group(title='Plugins options')
+groups['tokens'] = parser.add_argument_group(title='Tokens options')
 groups['packages'] = parser.add_argument_group(title='Packages options')
 groups['debug'] = parser.add_argument_group(title='Debugging options')
 groups['misc'] = parser.add_argument_group(title='Miscellaneous options')
@@ -393,7 +394,11 @@ groups['api'].add_argument(
 )
 groups['api'].add_argument(
     '--no-dash', '--nodash', action='store_true',
-    help = 'When starting the API, do not start the Web interface.',
+    help = 'When starting the API, do not start the Web Console dashboard.',
+)
+groups['api'].add_argument(
+    '--no-webterm', '--nowebterm', action='store_true',
+    help="When starting the API, do not start the Webterm.",
 )
 groups['api'].add_argument(
     '--private', '--private-mode', action='store_true',
@@ -419,12 +424,23 @@ groups['api'].add_argument(
     '--certfile', type=str,
     help = "Start the API server with this certfile (requires --keyfile).",
 )
+
 ### Plugins options
 groups['plugins'].add_argument(
     '-r', '--repository', '--repo', type=str,
     help="Meerschaum plugins repository to connect to. Specify an API label (default: 'mrsm')"
 )
 
+### Tokens options
+groups['tokens'].add_argument(
+    '--ttl-days', type=int,
+    help="For how many days should a token be valid.",
+)
+groups['tokens'].add_argument(
+    '--scopes', '--scope', nargs='+',
+    help="Which scope permissions to grant to a token.",
+)
+
 ### Packages options
 groups['packages'].add_argument(
     '--venv', type=str,

meerschaum/_internal/entry.py

@@ -15,7 +15,7 @@ import pathlib
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
-from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG as _STATIC_CONFIG
 
 _systemd_result_path = None
 if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
@@ -66,7 +66,7 @@ def entry(
         sysargs_has_api_executor_keys,
         get_pipeline_sysargs,
     )
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     if sysargs is None:
         sysargs = []
     if not isinstance(sysargs, list):
@@ -206,7 +206,7 @@ def entry_with_args(
     from meerschaum.actions import get_action
     from meerschaum._internal.arguments import remove_leading_action
     from meerschaum.utils.venv import active_venvs, deactivate_venv
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.typing import is_success_tuple
 
     if _patch_args:
@@ -325,9 +325,9 @@ def _do_action_wrapper(
                 import traceback
                 traceback.print_exception(type(e), e, e.__traceback__)
             result = False, (
-                f"Failed to execute `{action_name}` "
-                + "with exception:\n\n" +
-                f"{e}."
+                f"Failed to execute `{action_name.strip()}` "
+                + f"with `{type(e).__name__}`"
+                + (f':\n\n{e}' if str(e) else '.')
                 + (
                     "\n\nRun again with '--debug' to see a full stacktrace."
                     if not kw.get('debug', False) else ''

meerschaum/_internal/shell/Shell.py

@@ -35,7 +35,7 @@ _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
 from meerschaum.utils.warnings import warn
 from meerschaum.jobs import get_executor_keys_from_context
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum._internal.arguments._parse_arguments import (
     split_chained_sysargs,
     split_pipeline_sysargs,

meerschaum/_internal/static.py

@@ -0,0 +1,372 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define internal static config (formerly `meerschaum.config.static`).
+"""
+
+import os
+import random
+import string
+from typing import Dict, Any
+
+__all__ = ('STATIC_CONFIG',)
+
+_default_create_engine_args = {
+    #  'method': 'multi',
+    'pool_size': 5,
+    'max_overflow': 10,
+    'pool_recycle': 3600,
+    'connect_args': {},
+}
+_default_db_requirements = {
+    'username',
+    'password',
+    'host',
+    'database',
+}
+SERVER_ID: str = os.environ.get('MRSM_SERVER_ID', ''.join(random.sample(string.ascii_lowercase+string.digits, 6)))
+STATIC_CONFIG: Dict[str, Any] = {
+    'api': {
+        'endpoints': {
+            'index': '/',
+            'favicon': '/favicon.ico',
+            'plugins': '/plugins',
+            'pipes': '/pipes',
+            'metadata': '/metadata',
+            'actions': '/actions',
+            'jobs': '/jobs',
+            'logs': '/logs',
+            'users': '/users',
+            'tokens': '/tokens',
+            'login': '/login',
+            'connectors': '/connectors',
+            'version': '/version',
+            'chaining': '/chaining',
+            'websocket': '/ws',
+            'dash': '/dash',
+            'webterm': r'/webterm/{session_id}',
+            'webterm_websocket': r'/websocket/{session_id}',
+            'info': '/info',
+            'healthcheck': '/healthcheck',
+            'docs': '/docs',
+            'redoc': '/redoc',
+            'openapi': '/openapi.json',
+        },
+        'oauth': {
+            'token_expires_minutes': 720,
+        },
+        'webterm_job_name': '_webterm',
+        'default_timeout': 600,
+        'jobs': {
+            'stdin_message': 'MRSM_STDIN',
+            'stop_message': 'MRSM_STOP',
+            'metadata_cache_seconds': 5,
+            'temp_prefix': '.api-temp-',
+        },
+    },
+    'sql': {
+        'internal_schema': '_mrsm_internal',
+        'instance_schema': 'mrsm',
+        'default_create_engine_args': _default_create_engine_args,
+        'create_engine_flavors': {
+            'timescaledb': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'timescaledb-ha': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'postgresql': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'postgis': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'citus': {
+                'engine': 'postgresql+psycopg',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {},
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 5432,
+                },
+            },
+            'mssql': {
+                'engine': 'mssql+pyodbc',
+                'create_engine': {
+                    'fast_executemany': True,
+                    'use_insertmanyvalues': False,
+                    'isolation_level': 'AUTOCOMMIT',
+                    'use_setinputsizes': False,
+                    'pool_pre_ping': True,
+                    'ignore_no_transaction_on_rollback': True,
+                },
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': None,
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 1433,
+                    'options': (
+                        "driver=ODBC Driver 18 for SQL Server"
+                        "&UseFMTONLY=Yes"
+                        "&TrustServerCertificate=yes"
+                        "&Encrypt=no"
+                        "&MARS_Connection=yes"
+                    ),
+                },
+            },
+            'mysql': {
+                'engine': 'mysql+pymysql',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 3306,
+                },
+            },
+            'mariadb': {
+                'engine': 'mysql+pymysql',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 3306,
+                },
+            },
+            'oracle': {
+                'engine': 'oracle+oracledb',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': None,
+                },
+                'requirements': _default_db_requirements,
+                'defaults': {
+                    'port': 1521,
+                },
+            },
+            'sqlite': {
+                'engine': 'sqlite',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': {'database'},
+                'defaults': {},
+            },
+            'duckdb': {
+                'engine': 'duckdb',
+                'create_engine': {},
+                'omit_create_engine': {'ALL',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': '',
+                'defaults': {},
+            },
+            'cockroachdb': {
+                'engine': 'cockroachdb',
+                'omit_create_engine': {'method',},
+                'create_engine': _default_create_engine_args,
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': {'host'},
+                'defaults': {
+                    'port': 26257,
+                    'database': 'defaultdb',
+                    'username': 'root',
+                    'password': 'admin',
+                },
+            },
+        },
+    },
+    'valkey': {
+        'colon': '-_',
+    },
+    'environment': {
+        'config': 'MRSM_CONFIG',
+        'config_dir': 'MRSM_CONFIG_DIR',
+        'patch': 'MRSM_PATCH',
+        'root': 'MRSM_ROOT_DIR',
+        'plugins': 'MRSM_PLUGINS_DIR',
+        'venvs': 'MRSM_VENVS_DIR',
+        'runtime': 'MRSM_RUNTIME',
+        'work_dir': 'MRSM_WORK_DIR',
+        'user': 'MRSM_USER',
+        'dep_group': 'MRSM_DEP_GROUP',
+        'home': 'MRSM_HOME',
+        'src': 'MRSM_SRC',
+        'uid': 'MRSM_UID',
+        'gid': 'MRSM_GID',
+        'noask': 'MRSM_NOASK',
+        'noninteractive': 'MRSM_NONINTERACTIVE',
+        'id': 'MRSM_SERVER_ID',
+        'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
+        'systemd_delete_job': 'MRSM_SYSTEMD_DELETE_JOB',
+        'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
+        'prefix': 'MRSM_',
+    },
+    'config': {
+        'default_filetype': 'json',
+        'symlinks_key': '_symlinks',
+    },
+    'system': {
+        'arguments': {
+            'sub_decorators': (
+                '[',
+                ']'
+            ),
+            'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
+            'failure_key': '_argparse_exception',
+            'and_key': '+',
+            'escaped_and_key': '++',
+            'pipeline_key': ':',
+            'escaped_pipeline_key': '::',
+        },
+        'urls': {
+            'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',
+        },
+        'success': {
+            'ignore': (
+                'Success',
+                'success'
+                'Succeeded',
+                '',
+                None,
+            ),
+        },
+        'prompt': {
+            'web': False,
+        },
+        'fetch_pipes_keys': {
+            'negation_prefix': '_',
+        },
+    },
+    'connectors': {
+        'default_label': 'main',
+    },
+    'stack': {
+        'dollar_standin': '<DOLLAR>', ### Temporary replacement for docker-compose.yaml.
+    },
+    'users': {
+        'password_hash': {
+            'algorithm_name': 'sha256',
+            'salt_bytes': 16,
+            'schemes': [
+                'pbkdf2_sha256',
+            ],
+            'default': 'pbkdf2_sha256',
+            'pbkdf2_sha256__default_rounds': 1_000_000,
+        },
+        'min_username_length': 1,
+        'max_username_length': 60,
+        'min_password_length': 5,
+    },
+    'plugins': {
+        'repo_separator': '@',
+        'lock_sleep_total': 1.0,
+        'lock_sleep_increment': 0.1,
+    },
+    'pipes': {
+        'dtypes': {
+            'min_ratio_columns_changed_for_full_astype': 0.5,
+        },
+        'exists_timeout_seconds': 5.0,
+        'static_schema_cache_seconds': 60.0,
+        'max_bound_time_days': 36525,
+    },
+    'jobs': {
+        'check_restart_seconds': 1.0,
+    },
+    'tokens': {
+        'minimum_length': 24,
+        'maximum_length': 32,
+        'hash_rounds': 100_000,
+        'scopes': {
+            'pipes:read': "Read pipes' parameters and the contents of target tables.",
+            'pipes:write': "Update pipes' parameters and sync to target tables.",
+            'pipes:drop': "Drop target tables.",
+            'pipes:delete': "Delete pipes' parameters and drop target tables.",
+            'actions:execute': "Execute arbitrary actions.",
+            'connectors:read': "Read the available connectors.",
+            'jobs:read': "Read jobs' properties",
+            'jobs:write': "Write jobs' properties",
+            'jobs:execute': "Run jobs.",
+            'jobs:delete': "Delete jobs.",
+            'logs:read': "Read jobs' logs.",
+            'jobs:stop': "Stop running jobs.",
+            'jobs:pause': "Pause running jobs.",
+            'instance:read': "Read an instance's system-level metadata.",
+            'instance:chain': "Allow chaining API instances using the associated credentials.",
+            'plugins:write': "Register and update plugins' metadata.",
+            'plugins:read': "Read attributes of registered plugins.",
+            'plugins:delete': "Delete plugins (owned by user) from the repository.",
+            'users:read': "Read metadata about the associated account.",
+            'users:write': "Write metadata for the associated account.",
+            'users:register': "Register new user accounts.",
+            'users:delete': "Delete the associated user account (or other users for admins).",
+        },
+    },
+    'setup': {
+        'name': 'meerschaum',
+        'formal_name': 'Meerschaum',
+        'app_id': 'io.meerschaum',
+        'description': 'Sync Time-Series Pipes with Meerschaum',
+        'url': 'https://meerschaum.io',
+        'project_urls': {
+            'Documentation': 'https://docs.meerschaum.io',
+            'Changelog': 'https://meerschaum.io/news/changelog',
+            'GitHub': 'https://github.com/bmeares/Meerschaum',
+            'Homepage': 'https://meerschaum.io',
+            'Donate': 'https://github.com/sponsors/bmeares',
+            'Discord': 'https://discord.gg/8U8qMUjvcc',
+        },
+        'author': 'Bennett Meares',
+        'author_email': 'bennett.meares@gmail.com',
+        'maintainer_email': 'bennett.meares@gmail.com',
+        'license': 'Apache Software License 2.0',
+        'license_files': ('LICENSE',),
+    },
+}

meerschaum/actions/api.py

@@ -96,6 +96,7 @@ def _api_start(
     workers: Optional[int] = None,
     mrsm_instance: Optional[str] = None,
     no_dash: bool = False,
+    no_webterm: bool = False,
     no_auth: bool = False,
     private: bool = False,
     secure: bool = False,
@@ -166,7 +167,7 @@ def _api_start(
     )
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.config._environment import get_env_vars
-    from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
+    from meerschaum._internal.static import STATIC_CONFIG, SERVER_ID
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.pool import get_pool
 
@@ -244,6 +245,7 @@ def _api_start(
         'env_file': str(uvicorn_env_path.as_posix()),
         'mrsm_instance': mrsm_instance,
         'no_dash': no_dash,
+        'no_webterm': no_webterm or no_auth,
         'no_auth': no_auth,
         'private': private,
         'production': production,
@@ -263,7 +265,15 @@ def _api_start(
     if secure:
         cf['system']['api']['permissions']['actions']['non_admin'] = False
 
-    custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug', 'production']
+    custom_keys = [
+        'mrsm_instance',
+        'no_dash',
+        'no_webterm',
+        'no_auth',
+        'private',
+        'debug',
+        'production',
+    ]
 
     ### write config to a temporary file to communicate with uvicorn threads
     try:

meerschaum/actions/bootstrap.py

@@ -10,7 +10,7 @@ Functions for bootstrapping elements
 from __future__ import annotations
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
+from meerschaum.utils.typing import Union, Any, SuccessTuple, Optional, Tuple, List
 
 
 def bootstrap(
@@ -249,7 +249,7 @@ def _bootstrap_connectors(
     from meerschaum.config._edit import write_config
     from meerschaum.utils.formatting import pprint
     from meerschaum.utils.formatting._shell import clear_screen
-    from meerschaum.connectors import attributes as connector_attributes
+    from meerschaum.config._default import CONNECTOR_ATTRIBUTES
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.misc import is_int
 
@@ -314,7 +314,7 @@ def _bootstrap_connectors(
     cls_required_attrs = getattr(cls, 'REQUIRED_ATTRIBUTES', [])
     cls_optional_attrs = getattr(cls, 'OPTIONAL_ATTRIBUTES', [])
     cls_default_attrs = getattr(cls, 'DEFAULT_ATTRIBUTES', {})
-    type_attributes = connector_attributes.get(
+    type_attributes = CONNECTOR_ATTRIBUTES.get(
         _type,
         {
             'required': cls_required_attrs,
@@ -330,16 +330,16 @@ def _bootstrap_connectors(
                 f"Flavor for connector '{_type}:{_label}':",
                 sorted(list(type_attributes['flavors'])),
                 default = (
-                    'timescaledb'
-                    if 'timescaledb' in type_attributes['flavors']
+                    'postgresql'
+                    if 'postgresql' in type_attributes['flavors']
                     else None
                 )
             )
         except KeyboardInterrupt:
             return abort_tuple
         new_attributes['flavor'] = flavor
-        required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
-        optional = sorted(list(connector_attributes[_type]['flavors'][flavor].get('optional', {})))
+        required = sorted(list(CONNECTOR_ATTRIBUTES[_type]['flavors'][flavor]['requirements']))
+        optional = sorted(list(CONNECTOR_ATTRIBUTES[_type]['flavors'][flavor].get('optional', {})))
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))