meerschaum 2.4.0.dev0__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

meerschaum/api/routes/_actions.py

@@ -7,28 +7,13 @@ Execute Meerschaum Actions via the API
 """
 
 from __future__ import annotations
-import asyncio
-import traceback
-import shlex
-from functools import partial
-from datetime import datetime, timezone
-
-from fastapi import WebSocket, WebSocketDisconnect
-from websockets.exceptions import ConnectionClosedError
-
-from meerschaum.utils.misc import generate_password
-from meerschaum.jobs import Job
-from meerschaum.utils.warnings import warn
-from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
+
+from meerschaum.utils.typing import SuccessTuple, List, Dict, Any
 from meerschaum.api import (
     fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
 )
 from meerschaum.actions import actions
-import meerschaum.core
 from meerschaum.config import get_config
-from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
-from meerschaum.api.routes._jobs import clean_sysargs
-from meerschaum.jobs._Job import StopMonitoringLogs
 
 actions_endpoint = endpoints['actions']
 
@@ -82,7 +67,7 @@ def do_action_legacy(
     ----------
     action: str
         The action to perform.
-        
+
     keywords: Dict[str, Any]
         The keywords dictionary to pass to the action.
 
@@ -91,7 +76,6 @@ def do_action_legacy(
     A `SuccessTuple`.
     """
     if curr_user is not None and curr_user.type != 'admin':
-        from meerschaum.config import get_config
         allow_non_admin = get_config(
             'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
         )
@@ -103,7 +87,7 @@ def do_action_legacy(
                 + "and search for 'permissions'. "
                 + "\nUnder the keys 'api:permissions:actions', "
                 + "you can allow non-admin users to perform actions."
-        )
+            )
 
     if action not in actions:
         return False, f"Invalid action '{action}'."

meerschaum/api/routes/_jobs.py

@@ -9,7 +9,6 @@ Manage jobs via the Meerschaum API.
 from __future__ import annotations
 
 import os
-import select
 import asyncio
 import traceback
 from datetime import datetime
@@ -18,12 +17,10 @@ from functools import partial
 
 from fastapi import WebSocket, WebSocketDisconnect
 
-from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union
 from meerschaum.jobs import (
     get_jobs as _get_jobs,
     Job,
-    StopMonitoringLogs,
-    get_executor_keys_from_context,
 )
 from meerschaum.utils.warnings import warn
 
@@ -32,14 +29,13 @@ from meerschaum.api import (
     app,
     endpoints,
     manager,
-    debug,
     no_auth,
-    private,
 )
 from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
 EXECUTOR_KEYS: str = 'local'
 
 
@@ -48,7 +44,8 @@ def _get_job(name: str):
     if systemd_job.exists():
         return systemd_job
 
-    return Job(name, executor_keys=EXECUTOR_KEYS)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -150,6 +147,10 @@ def create_job(
         name,
         clean_sysargs(sysargs),
         executor_keys=EXECUTOR_KEYS,
+        env={
+            NONINTERACTIVE_ENV: '1',
+            **dict(os.environ)
+        },
         _properties=properties,
     )
     if job.exists():

meerschaum/api/routes/_webterm.py

@@ -8,10 +8,9 @@ Routes to the Webterm proxy.
 
 import asyncio
 from meerschaum.utils.typing import Optional
-from meerschaum.api import app, no_auth, manager, endpoints
+from meerschaum.api import app, endpoints
 from meerschaum.utils.packages import attempt_import
-from meerschaum.api.dash import active_sessions
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
 import starlette
 httpx = attempt_import('httpx')
@@ -25,9 +24,9 @@ PlainTextResponse = fastapi_responses.PlainTextResponse
 
 @app.get(endpoints['webterm'], tags=["Webterm"])
 async def get_webterm(
-        request: Request,
-        s: Optional[str] = None,
-    ) -> HTMLResponse:
+    request: Request,
+    s: Optional[str] = None,
+) -> HTMLResponse:
     """
     Get the main HTML template for the Webterm.
     """

meerschaum/config/_default.py

@@ -96,6 +96,10 @@ default_system_config = {
             'proxy_headers': True,
             'forwarded_allow_ips': '*',
         },
+        'cache': {
+            'connector': 'valkey:main',
+            'session_expires_minutes': 43200,
+        },
         'permissions':       {
             'registration': {
                 'users': True,
@@ -105,7 +109,7 @@ default_system_config = {
             'actions': {
                 'non_admin': True,
             },
-            'chaining' : {
+            'chaining': {
                 'insecure_parent_instance': False,
                 'child_apis': False,
             },
@@ -120,6 +124,7 @@ default_system_config = {
         'inplace_sync': True,
         'uv_pip': True,
         'systemd_healthcheck': False,
+        'valkey_session_cache': True,
     },
 }
 default_pipes_config = {

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.0.dev0"
+__version__ = "2.4.0rc1"

meerschaum/config/stack/__init__.py

@@ -52,8 +52,8 @@ env_dict = {
 ### apply patch to host config to change hostname to the Docker service name
 env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
-        'meerschaum' : 'MRSM{!meerschaum}',
-        'system' : 'MRSM{!system}',
+        'meerschaum': 'MRSM{!meerschaum}',
+        'system': 'MRSM{!system}',
     },
     indent = 4,
 ).replace(
@@ -82,12 +82,14 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
                         'port': 5432,
                     },
                     'local': {
-                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
+                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db',
                     },
                 },
                 'valkey': {
-                    'host': valkey_host,
-                    'port': 6379,
+                    'main': {
+                        'host': valkey_host,
+                        'port': 6379,
+                    },
                 },
             },
         },
@@ -133,7 +135,7 @@ default_docker_compose_config = {
             'ports': [
                 f'{db_port}:5432',
             ],
-            'hostname': f'{db_hostname}',
+            'hostname': db_hostname,
             'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
@@ -178,7 +180,7 @@ default_docker_compose_config = {
             ],
         },
         'valkey': {
-            'image': 'bitnami/valkey',
+            'image': 'bitnami/valkey:latest',
             'restart': 'always',
             'environment': {
                 'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',

meerschaum/config/static/__init__.py

@@ -53,6 +53,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         'internal_schema': '_mrsm_internal',
         'instance_schema': 'mrsm',
     },
+    'valkey': {
+        'colon': '-_',
+    },
     'environment': {
         'config': 'MRSM_CONFIG',
         'config_dir': 'MRSM_CONFIG_DIR',
@@ -69,6 +72,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'uid': 'MRSM_UID',
         'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
+        'noninteractive': 'MRSM_NONINTERACTIVE',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
         'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',

meerschaum/connectors/__init__.py

@@ -14,13 +14,13 @@ For ease of use, you can also import from the root `meerschaum` module:
 from __future__ import annotations
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List, Dict
-from meerschaum.utils.threading import Lock, RLock
-from meerschaum.utils.warnings import error, warn
+from meerschaum.utils.typing import Any, Union, List, Dict
+from meerschaum.utils.threading import RLock
+from meerschaum.utils.warnings import warn
 
-from meerschaum.connectors.Connector import Connector, InvalidAttributesError
-from meerschaum.connectors.sql.SQLConnector import SQLConnector
-from meerschaum.connectors.api.APIConnector import APIConnector
+from meerschaum.connectors._Connector import Connector, InvalidAttributesError
+from meerschaum.connectors.sql._SQLConnector import SQLConnector
+from meerschaum.connectors.api._APIConnector import APIConnector
 from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
 
 __all__ = (
@@ -31,6 +31,9 @@ __all__ = (
     "get_connector",
     "is_connected",
     "poll",
+    "api",
+    "sql",
+    "valkey",
 )
 
 ### store connectors partitioned by
@@ -54,7 +57,10 @@ attributes: Dict[str, Dict[str, Any]] = {
         'required': [
             'host',
             'username',
-            'password'
+            'password',
+        ],
+        'optional': [
+            'port',
         ],
         'default': {
             'protocol': 'http',
@@ -263,11 +269,11 @@ def is_connected(keys: str, **kw) -> bool:
 
     try:
         typ, label = keys.split(':')
-    except Exception as e:
+    except Exception:
         return False
     if typ not in instance_types:
         return False
-    if not (label in connectors.get(typ, {})):
+    if label not in connectors.get(typ, {}):
         return False
 
     from meerschaum.connectors.parse import parse_instance_keys

meerschaum/connectors/api/{APIConnector.py → _APIConnector.py}

@@ -7,7 +7,7 @@ Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_inst
 """
 
 from datetime import datetime, timedelta, timezone
-from meerschaum.utils.typing import Optional
+from meerschaum.utils.typing import Optional, List
 from meerschaum.connectors import Connector
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.packages import attempt_import
@@ -24,6 +24,8 @@ class APIConnector(Connector):
     IS_INSTANCE: bool = True
     IS_THREAD_SAFE: bool = False
 
+    OPTIONAL_ATTRIBUTES: List[str] = ['port']
+
     from ._request import (
         make_request,
         get,

meerschaum/connectors/api/__init__.py

@@ -6,5 +6,6 @@
 Interact with the Meerschaum API (send commands, pull data, etc.)
 """
 
-from meerschaum.connectors.api.APIConnector import APIConnector
+from meerschaum.connectors.api._APIConnector import APIConnector
 
+__all__ = ('APIConnector',)

meerschaum/connectors/parse.py

@@ -7,7 +7,10 @@ Utility functions for parsing connector keys.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Any, Union, Optional, Dict, Tuple
+
 
 def parse_connector_keys(
     keys: str,
@@ -15,20 +18,20 @@ def parse_connector_keys(
     as_tuple: bool = False,
     **kw:  Any
 ) -> (
-        Union[
-            meerschaum.connectors.Connector,
-            Dict[str, Any],
-            Tuple[
-                Union[
-                    meerschaum.connectors.Connector,
-                    Dict[str, Any],
-                    None
-                ],
-                str
+    Union[
+        mrsm.connectors.Connector,
+        Dict[str, Any],
+        Tuple[
+            Union[
+                mrsm.connectors.Connector,
+                Dict[str, Any],
+                None,
             ],
-            None
-        ]
-    ):
+            str
+        ],
+        None
+    ]
+):
     """
     Convenience function for parsing connector keys and returning Connector objects.
 
@@ -95,13 +98,12 @@ def parse_instance_keys(
     """
     Parse the Meerschaum instance value into a Connector object.
     """
-    from meerschaum.utils.warnings import warn
     from meerschaum.config import get_config
 
     if keys is None:
         keys = get_config('meerschaum', 'instance')
     keys = str(keys)
-    
+
     return parse_connector_keys(keys, construct=construct, as_tuple=as_tuple, **kw)
 
 

meerschaum/connectors/sql/__init__.py

@@ -6,4 +6,6 @@
 Subpackage for SQLConnector subclass
 """
 
-from meerschaum.connectors.sql.SQLConnector import SQLConnector
+from meerschaum.connectors.sql._SQLConnector import SQLConnector
+
+__all__ = ('SQLConnector',)

meerschaum/connectors/sql/_pipes.py

@@ -821,7 +821,8 @@ def get_pipe_data_query(
     begin: Union[datetime, int, str, None] = None,
     end: Union[datetime, int, str, None] = None,
     params: Optional[Dict[str, Any]] = None,
-    order: str = 'asc',
+    order: Optional[str] = 'asc',
+    sort_datetimes: bool = False,
     limit: Optional[int] = None,
     begin_add_minutes: int = 0,
     end_add_minutes: int = 0,
@@ -854,10 +855,13 @@ def get_pipe_data_query(
         Additional parameters to filter by.
         See `meerschaum.connectors.sql.build_where`.
 
-    order: Optional[str], default 'asc'
+    order: Optional[str], default None
         The selection order for all of the indices in the query.
         If `None`, omit the `ORDER BY` clause.
 
+    sort_datetimes: bool, default False
+        Alias for `order='desc'`.
+
     limit: Optional[int], default None
         If specified, limit the number of rows retrieved to this value.
 
@@ -880,7 +884,6 @@ def get_pipe_data_query(
     -------
     A `SELECT` query to retrieve a pipe's data.
     """
-    import json
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.misc import items_str
     from meerschaum.utils.sql import sql_item_name, dateadd_str
@@ -895,6 +898,9 @@ def get_pipe_data_query(
     if omit_columns:
         select_columns = [col for col in select_columns if col not in omit_columns]
 
+    if order is None and sort_datetimes:
+        order = 'desc'
+
     if begin == '':
         begin = pipe.get_sync_time(debug=debug)
         backtrack_interval = pipe.get_backtrack_interval(debug=debug)
@@ -960,10 +966,10 @@ def get_pipe_data_query(
     is_dt_bound = False
     if begin is not None and _dt in existing_cols:
         begin_da = dateadd_str(
-            flavor = self.flavor,
-            datepart = 'minute',
-            number = begin_add_minutes,
-            begin = begin
+            flavor=self.flavor,
+            datepart='minute',
+            number=begin_add_minutes,
+            begin=begin,
         )
         where += f"{dt} >= {begin_da}" + (" AND " if end is not None else "")
         is_dt_bound = True
@@ -972,10 +978,10 @@ def get_pipe_data_query(
         if 'int' in str(type(end)).lower() and end == begin:
             end += 1
         end_da = dateadd_str(
-            flavor = self.flavor,
-            datepart = 'minute',
-            number = end_add_minutes,
-            begin = end
+            flavor=self.flavor,
+            datepart='minute',
+            number=end_add_minutes,
+            begin=end
         )
         where += f"{dt} < {end_da}"
         is_dt_bound = True
@@ -1599,45 +1605,39 @@ def sync_pipe_inplace(
 
     backtrack_def = self.get_pipe_data_query(
         pipe,
-        begin = begin,
-        end = end,
-        begin_add_minutes = 0,
-        end_add_minutes = 1,
-        params = params,
-        debug = debug,
-        order = None,
+        begin=begin,
+        end=end,
+        begin_add_minutes=0,
+        end_add_minutes=1,
+        params=params,
+        debug=debug,
+        order=None,
     )
 
-    select_backtrack_query = format_cte_subquery(
-        backtrack_def,
-        self.flavor,
-        sub_name = 'backtrack_def',
-    )
     create_backtrack_query = get_create_table_query(
         backtrack_def,
         temp_tables['backtrack'],
         self.flavor,
-        schema = internal_schema,
+        schema=internal_schema,
     )
     (create_backtrack_success, create_backtrack_msg), create_backtrack_results = session_execute(
         session,
         create_backtrack_query,
-        with_results = True,
-        debug = debug,
+        with_results=True,
+        debug=debug,
     ) if not upsert else (True, "Success"), None
 
     if not create_backtrack_success:
         _ = clean_up_temp_tables()
         return create_backtrack_success, create_backtrack_msg
-    bactrack_count = create_backtrack_results[0].rowcount if create_backtrack_results else 0
 
     backtrack_cols_types = get_table_cols_types(
         temp_tables['backtrack'],
-        connectable = connectable,
-        flavor = self.flavor,
-        schema = internal_schema,
-        database = database,
-        debug = debug,
+        connectable=connectable,
+        flavor=self.flavor,
+        schema=internal_schema,
+        database=database,
+        debug=debug,
     ) if not upsert else new_cols_types
 
     common_cols = [col for col in new_cols if col in backtrack_cols_types]
@@ -1663,7 +1663,7 @@ def sync_pipe_inplace(
     )
 
     select_delta_query = (
-        f"SELECT\n"
+        "SELECT\n"
         + null_replace_new_cols_str + "\n"
         + f"\nFROM {temp_table_names['new']}\n"
         + f"LEFT OUTER JOIN {temp_table_names['backtrack']}\nON\n"
@@ -1693,12 +1693,12 @@ def sync_pipe_inplace(
         select_delta_query,
         temp_tables['delta'],
         self.flavor,
-        schema = internal_schema,
+        schema=internal_schema,
     )
     create_delta_success, create_delta_msg = session_execute(
         session,
         create_delta_query,
-        debug = debug,
+        debug=debug,
     ) if not upsert else (True, "Success")
     if not create_delta_success:
         _ = clean_up_temp_tables()
@@ -1707,10 +1707,10 @@ def sync_pipe_inplace(
     delta_cols_types = get_table_cols_types(
         temp_tables['delta'],
         connectable = connectable,
-        flavor = self.flavor,
-        schema = internal_schema,
-        database = database,
-        debug = debug,
+        flavor=self.flavor,
+        schema=internal_schema,
+        database=database,
+        debug=debug,
     ) if not upsert else new_cols_types
 
     ### This is a weird bug on SQLite.

meerschaum/connectors/valkey/{ValkeyConnector.py → _ValkeyConnector.py}

@@ -11,9 +11,8 @@ from datetime import datetime, timezone
 
 import meerschaum as mrsm
 from meerschaum.connectors import Connector, make_connector
-from meerschaum.utils.typing import List, Dict, Any, Iterator, Optional, Union
-from meerschaum.utils.warnings import warn, dprint
-from meerschaum.utils.misc import json_serialize_datetime
+from meerschaum.utils.typing import List, Dict, Any, Optional, Union
+from meerschaum.utils.warnings import dprint
 
 
 @make_connector
@@ -141,11 +140,11 @@ class ValkeyConnector(Connector):
 
         return uri
 
-    def set(self, key: str, value: Any) -> None:
+    def set(self, key: str, value: Any, **kwargs: Any) -> None:
         """
         Set the `key` to `value`.
         """
-        return self.client.set(key, value)
+        return self.client.set(key, value, **kwargs)
 
     def get(self, key: str) -> Union[str, None]:
         """
@@ -243,6 +242,7 @@ class ValkeyConnector(Connector):
         -------
         The current index counter value (how many docs have been pushed).
         """
+        from meerschaum.utils.misc import json_serialize_datetime
         table_name = self.quote_table(table)
         datetime_column_key = self.get_datetime_column_key(table)
         remote_datetime_column = self.get(datetime_column_key)

meerschaum/connectors/valkey/__init__.py

@@ -5,4 +5,6 @@
 Import the `ValkeyConnector`.
 """
 
-from meerschaum.connectors.valkey.ValkeyConnector import ValkeyConnector
+from meerschaum.connectors.valkey._ValkeyConnector import ValkeyConnector
+
+__all__ = ('ValkeyConnector',)

meerschaum/connectors/valkey/_pipes.py

@@ -6,15 +6,17 @@ Define pipes methods for `ValkeyConnector`.
 """
 
 import json
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, Union, Optional, Dict, List, Tuple
 from meerschaum.utils.misc import json_serialize_datetime, string_to_dict
-from meerschaum.utils.warnings import dprint, warn
+from meerschaum.utils.warnings import warn
+from meerschaum.config.static import STATIC_CONFIG
 
 PIPES_TABLE: str = 'mrsm_pipes'
 PIPES_COUNTER: str = 'mrsm_pipes:counter'
+COLON: str = STATIC_CONFIG['valkey']['colon']
 
 
 def get_pipe_key(pipe: mrsm.Pipe) -> str:
@@ -88,7 +90,7 @@ def get_document_key(
     indices_str = ((table_name + ':indices:') if table_name else '') + ','.join(
         sorted(
             [
-                f'{key}:{val}'
+                f'{key}{COLON}{val}'
                 for key, val in index_vals.items()
             ]
         )
@@ -413,7 +415,7 @@ def get_pipe_data(
     table_name = self.quote_table(pipe.target)
     indices = [col for col in pipe.columns.values() if col]
     ix_docs = [
-        string_to_dict(doc['ix'])
+        string_to_dict(doc['ix'].replace(COLON, ':'))
         for doc in self.read_docs(
             pipe.target,
             begin=begin,
@@ -505,6 +507,7 @@ def sync_pipe(
     is_dask = 'dask' in df.__module__
     if is_dask:
         df = df.compute()
+    upsert = pipe.parameters.get('upsert', False)
 
     def _serialize_indices_docs(_docs):
         return [
@@ -546,16 +549,18 @@ def sync_pipe(
             if not edit_success:
                 return edit_success, edit_msg
 
-    #  df = pipe.enforce_dtypes(df, debug=debug)
-
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(df, include_unchanged_columns=True, debug=debug)
-        if check_existing
+        if check_existing and not upsert
         else (df, None, df)
     )
     num_insert = len(unseen_df) if unseen_df is not None else 0
     num_update = len(update_df) if update_df is not None else 0
-    msg = f"Inserted {num_insert}, updated {num_update} rows."
+    msg = (
+        f"Inserted {num_insert}, updated {num_update} rows."
+        if not upsert
+        else f"Upserted {num_insert} rows."
+    )
     if len(delta_df) == 0:
         return True, msg