meerschaum 2.4.0rc2__py3-none-any.whl → 2.4.2__py3-none-any.whl

meerschaum/actions/bootstrap.py

@@ -336,7 +336,7 @@ def _bootstrap_connectors(
             return abort_tuple
         new_attributes['flavor'] = flavor
         required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
-        required = sorted(list(connector_attributes[_type]['flavors'][flavor]['optional']))
+        optional = sorted(list(connector_attributes[_type]['flavors'][flavor].get('optional', {})))
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))

meerschaum/api/dash/callbacks/dashboard.py

@@ -214,6 +214,13 @@ def update_content(*args):
     trigger = ctx.triggered[0]['prop_id'].split('.')[0] if not trigger else trigger
 
     session_data = args[-1]
+    mrsm_location_href = session_data.get('mrsm-location.href', None)
+    if (
+        initial_load
+        and mrsm_location_href
+        and not mrsm_location_href.rstrip('/').endswith('/dash')
+    ):
+        raise PreventUpdate
 
     ### NOTE: functions MUST return a list of content and a list of alerts
     triggers = {

meerschaum/api/dash/callbacks/pipes.py

@@ -5,7 +5,9 @@
 Define callbacks for the `/dash/pipes/` page.
 """
 
-from dash.dependencies import Input, Output
+from urllib.parse import parse_qs
+
+from dash.dependencies import Input, Output, State
 from dash import no_update
 
 import meerschaum as mrsm
@@ -13,17 +15,28 @@ from meerschaum.api.dash import dash_app
 from meerschaum.api.dash.pipes import build_pipe_card
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.utils.packages import import_html, import_dcc
+from meerschaum.api.dash.sessions import is_session_authenticated
+from meerschaum.utils.typing import Optional, Dict, Any
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 
 
 @dash_app.callback(
     Output('pipe-output-div', 'children'),
     Input('pipes-location', 'pathname'),
+    State('pipes-location', 'search'),
+    State('session-store', 'data'),
 )
-def render_page_from_url(pathname):
+def render_page_from_url(
+    pathname: str,
+    pipe_search: str,
+    session_data: Optional[Dict[str, Any]],
+):
     if not str(pathname).startswith('/dash/pipes'):
         return no_update
 
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+
     keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
     if len(keys) not in (2, 3):
         return no_update
@@ -31,11 +44,12 @@ def render_page_from_url(pathname):
     ck = keys[0]
     mk = keys[1]
     lk = keys[2] if len(keys) == 3 else None
+    query_params = parse_qs(pipe_search.lstrip('?')) if pipe_search else {}
+    instance = query_params.get('instance', [None])[0]
 
-    pipe = mrsm.Pipe(ck, mk, lk)
-    ### TODO Check if logged in
+    pipe = mrsm.Pipe(ck, mk, lk, instance=instance)
     return [
         html.Br(),
-        build_pipe_card(pipe, authenticated=False),
+        build_pipe_card(pipe, authenticated=authenticated, include_manage=False),
         html.Br(),
     ]

meerschaum/api/dash/pipes.py

@@ -10,6 +10,8 @@ from __future__ import annotations
 import json
 import shlex
 from textwrap import dedent
+from urllib.parse import urlencode
+
 from dash.dependencies import Input, Output, State
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
 from meerschaum.utils.misc import string_to_dict
@@ -30,6 +32,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 humanfriendly = attempt_import('humanfriendly', check_update=CHECK_UPDATE)
 pd = import_pandas()
 
+
 def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, None]:
     """
     Return a `meerschaum.Pipe` object from a dynamic object with an
@@ -105,6 +108,7 @@ def pipes_from_state(
 def build_pipe_card(
     pipe: mrsm.Pipe,
     authenticated: bool = False,
+    include_manage: bool = True,
     _build_children_num: int = 10,
 ) -> 'dbc.Card':
     """
@@ -118,6 +122,9 @@ def build_pipe_card(
     authenticated: bool, default False
         If `True`, allow editing functionality to the card.
 
+    include_manage: bool, default True
+        If `True` and `authenticated` is `True`, include the "Manage" dropdown.
+
     Returns
     -------
     A dash bootstrap components Card representation of the pipe.
@@ -184,7 +191,7 @@ def build_pipe_card(
                         size='sm',
                         color='secondary',
                     )
-                ) if authenticated else [],
+                ) if authenticated and include_manage else [],
                 width=2,
             ),
             dbc.Col(width=6),
@@ -217,8 +224,16 @@ def build_pipe_card(
 
     ]
 
+    query_params = {}
+    default_instance = get_config('meerschaum', 'instance')
+    if pipe.instance_keys != default_instance:
+        query_params['instance'] = pipe.instance_keys
     pipe_url = (
-        f"/dash/pipes/{pipe.connector_keys}/{pipe.metric_key}/{pipe.location_key}"
+        f"/dash/pipes/"
+        + f"{pipe.connector_keys}/"
+        + f"{pipe.metric_key}/"
+        + (f"{pipe.location_key}" if pipe.location_key is not None else '')
+        + (f"?{urlencode(query_params)}" if query_params else "")
     )
 
     card_header_children = dbc.Row(

meerschaum/config/__init__.py

@@ -79,15 +79,15 @@ def set_config(cf: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def get_config(
-        *keys: str,
-        patch: bool = True,
-        substitute: bool = True,
-        sync_files: bool = True,
-        write_missing: bool = True,
-        as_tuple: bool = False,
-        warn: bool = True,
-        debug: bool = False
-    ) -> Any:
+    *keys: str,
+    patch: bool = True,
+    substitute: bool = True,
+    sync_files: bool = True,
+    write_missing: bool = True,
+    as_tuple: bool = False,
+    warn: bool = True,
+    debug: bool = False
+) -> Any:
     """
     Return the Meerschaum configuration dictionary.
     If positional arguments are provided, index by the keys.

meerschaum/config/_read_config.py

@@ -10,12 +10,12 @@ from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
 from meerschaum.config import get_config
 
 def read_config(
-        directory: Optional[str] = None,
-        keys: Optional[List[str]] = None,
-        write_missing : bool = True,
-        substitute : bool = True,
-        with_filenames : bool = False,
-    ) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[str]]]:
+    directory: Optional[str] = None,
+    keys: Optional[List[str]] = None,
+    write_missing : bool = True,
+    substitute : bool = True,
+    with_filenames : bool = False,
+) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[str]]]:
     """
     Read the configuration directory.
 
@@ -313,10 +313,10 @@ def search_and_substitute_config(
         try:
             valid, value = get_config(
                 *keys,
-                substitute = False,
-                as_tuple = True,
-                write_missing = False,
-                sync_files = False,
+                substitute=False,
+                as_tuple=True,
+                write_missing=False,
+                sync_files=False,
             )
         except Exception as e:
             import traceback

meerschaum/config/_sync.py

@@ -7,17 +7,20 @@ Synchronize across config files
 """
 
 from __future__ import annotations
+import pathlib
 from meerschaum.utils.typing import Optional, List, Tuple
 
+
 def sync_yaml_configs(
-        keys: List[str],
-        sub_path: pathlib.Path,
-        substitute: bool = True,
-        permissions: Optional[int] = None,
-        replace_tuples: Optional[List[Tuple[str, str]]] = None,
-    ) -> None:
-    """Synchronize sub-configuration with main configuration file.
-    
+    keys: List[str],
+    sub_path: pathlib.Path,
+    substitute: bool = True,
+    permissions: Optional[int] = None,
+    replace_tuples: Optional[List[Tuple[str, str]]] = None,
+) -> None:
+    """
+    Synchronize sub-configuration with main configuration file.
+
     Parameters
     ----------
     keys: List[str]
@@ -84,12 +87,14 @@ def sync_yaml_configs(
     new_path = sub_path
 
     ### write changes
+    new_path.parent.mkdir(exist_ok=True, parents=True)
     with open(new_path, 'w+', encoding='utf-8') as f:
         f.write(new_header)
         f.write(new_config_text)
     if permissions is not None:
         os.chmod(new_path, permissions)
 
+
 def sync_files(keys: Optional[List[str]] = None):
     if keys is None:
         keys = []
@@ -110,21 +115,23 @@ def sync_files(keys: Optional[List[str]] = None):
         sync_yaml_configs(
             ['stack', STACK_COMPOSE_FILENAME],
             STACK_COMPOSE_PATH,
-            substitute = True,
-            replace_tuples = [('$', '$$'), ('<DOLLAR>', '$')],
+            substitute=True,
+            replace_tuples=[
+                ('$', '$$'),
+                ('<DOLLAR>', '$'),
+            ],
         )
         sync_yaml_configs(
             ['stack', 'grafana', 'datasource'],
             GRAFANA_DATASOURCE_PATH,
-            substitute = True,
+            substitute=True,
         )
         sync_yaml_configs(
             ['stack', 'grafana', 'dashboard'],
             GRAFANA_DASHBOARD_PATH,
-            substitute = True,
+            substitute=True,
         )
 
-
     key_functions = {
         'stack': _stack,
     }

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.0rc2"
+__version__ = "2.4.2"

meerschaum/config/stack/__init__.py

@@ -55,7 +55,7 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
         'meerschaum': 'MRSM{!meerschaum}',
         'system': 'MRSM{!system}',
     },
-    indent = 4,
+    indent=4,
 ).replace(
     '"MRSM{!system}"', 'MRSM{!system}'
 ).replace(

meerschaum/connectors/api/_APIConnector.py

@@ -168,7 +168,8 @@ class APIConnector(Connector):
     @property
     def session(self):
         if self._session is None:
-            requests = attempt_import('requests')
+            certifi = attempt_import('certifi', lazy=False)
+            requests = attempt_import('requests', lazy=False)
             if requests:
                 self._session = requests.Session()
             if self._session is None:

meerschaum/connectors/sql/_create_engine.py

@@ -27,39 +27,39 @@ default_create_engine_args = {
     'connect_args': {},
 }
 flavor_configs = {
-    'timescaledb'      : {
-        'engine'       : 'postgresql+psycopg',
-        'create_engine' : default_create_engine_args,
+    'timescaledb': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
-        'to_sql' : {},
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 5432,
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
         },
     },
-    'postgresql'         : {
-        'engine'       : 'postgresql+psycopg',
-        'create_engine' : default_create_engine_args,
+    'postgresql': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
-        'to_sql' : {},
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 5432,
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
         },
     },
-    'citus'            : {
-        'engine'       : 'postgresql+psycopg',
-        'create_engine' : default_create_engine_args,
+    'citus': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
-        'to_sql' : {},
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 5432,
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
         },
     },
-    'mssql'            : {
-        'engine'       : 'mssql+pyodbc',
-        'create_engine' : {
+    'mssql': {
+        'engine': 'mssql+pyodbc',
+        'create_engine': {
             'fast_executemany': True,
             'isolation_level': 'AUTOCOMMIT',
             'use_setinputsizes': False,
@@ -68,84 +68,81 @@ flavor_configs = {
         'to_sql': {
             'method': None,
         },
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 1433,
-            'options'  : {
-                'driver'   : 'ODBC Driver 17 for SQL Server',
-                'UseFMTONLY': 'Yes',
-            },
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 1433,
+            'options': "driver=ODBC Driver 17 for SQL Server&UseFMTONLY=Yes",
         },
     },
-    'mysql'            : {
-        'engine'       : 'mysql+pymysql',
-        'create_engine' : default_create_engine_args,
+    'mysql': {
+        'engine': 'mysql+pymysql',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql': {
             'method': 'multi',
         },
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 3306,
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 3306,
         },
     },
-    'mariadb'          : {
-        'engine'       : 'mysql+pymysql',
-        'create_engine' : default_create_engine_args,
+    'mariadb': {
+        'engine': 'mysql+pymysql',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql': {
             'method': 'multi',
         },
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 3306,
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 3306,
         },
     },
-    'oracle'           : {
-        'engine'       : 'oracle+cx_oracle',
-        'create_engine' : default_create_engine_args,
+    'oracle': {
+        'engine': 'oracle+cx_oracle',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql': {
             'method': None,
         },
-        'requirements' : default_requirements,
-        'defaults'     : {
-            'port'     : 1521,
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 1521,
         },
     },
-    'sqlite'           : {
-        'engine'       : 'sqlite',
-        'create_engine' : default_create_engine_args,
+    'sqlite': {
+        'engine': 'sqlite',
+        'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql': {
             'method': 'multi',
         },
-        'requirements' : {'database'},
-        'defaults'     : {},
+        'requirements': {'database'},
+        'defaults': {},
     },
-    'duckdb' : {
-        'engine' : 'duckdb',
-        'create_engine' : {},
+    'duckdb': {
+        'engine': 'duckdb',
+        'create_engine': {},
         'omit_create_engine': {'ALL',},
         'to_sql': {
             'method': 'multi',
         },
-        'requirements' : '',
-        'defaults' : {},
+        'requirements': '',
+        'defaults': {},
     },
-    'cockroachdb'      : {
-        'engine'       : 'cockroachdb',
+    'cockroachdb': {
+        'engine': 'cockroachdb',
         'omit_create_engine': {'method',},
         'create_engine': default_create_engine_args,
         'to_sql': {
             'method': 'multi',
         },
-        'requirements' : {'host'},
-        'defaults'     : {
-            'port'     : 26257,
-            'database' : 'defaultdb',
-            'username' : 'root',
-            'password' : 'admin',
+        'requirements': {'host'},
+        'defaults': {
+            'port': 26257,
+            'database': 'defaultdb',
+            'username': 'root',
+            'password': 'admin',
         },
     },
 }

meerschaum/connectors/sql/_pipes.py

@@ -909,7 +909,7 @@ def get_pipe_data_query(
 
     cols_names = [sql_item_name(col, self.flavor, None) for col in select_columns]
     select_cols_str = (
-        'SELECT\n'
+        'SELECT\n    '
         + ',\n    '.join(
             [
                 (
@@ -953,14 +953,14 @@ def get_pipe_data_query(
                 warn(
                     f"No datetime could be determined for {pipe}."
                     + "\n    Ignoring begin and end...",
-                    stack = False,
+                    stack=False,
                 )
                 begin, end = None, None
             else:
                 warn(
                     f"A datetime wasn't specified for {pipe}.\n"
                     + f"    Using column \"{_dt}\" for datetime bounds...",
-                    stack = False,
+                    stack=False,
                 )
 
     is_dt_bound = False
@@ -1014,9 +1014,12 @@ def get_pipe_data_query(
 
     if isinstance(limit, int):
         if self.flavor == 'mssql':
-            query = f'SELECT TOP {limit} ' + query[len("SELECT *"):]
+            query = f'SELECT TOP {limit}\n' + query[len("SELECT "):]
         elif self.flavor == 'oracle':
-            query = f"SELECT * FROM (\n  {query}\n)\nWHERE ROWNUM = 1"
+            query = (
+                f"SELECT * FROM (\n  {query}\n)\n"
+                + f"WHERE ROWNUM IN ({', '.join([str(i) for i in range(1, limit+1)])})"
+            )
         else:
             query += f"\nLIMIT {limit}"
     

meerschaum/core/Pipe/_data.py

@@ -106,7 +106,7 @@ def get_data(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import iterate_chunks, items_str
     from meerschaum.utils.dtypes import to_pandas_dtype
-    from meerschaum.utils.dataframe import add_missing_cols_to_df
+    from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
@@ -122,6 +122,8 @@ def get_data(
     as_iterator = as_iterator or as_chunks
 
     def _sort_df(_df):
+        if df_is_chunk_generator(_df):
+            return _df
         dt_col = self.columns.get('datetime', None)
         indices = [] if dt_col not in _df.columns else [dt_col]
         non_dt_cols = [
@@ -130,12 +132,19 @@ def get_data(
             if col_ix != 'datetime' and col in _df.columns
         ]
         indices.extend(non_dt_cols)
-        _df.sort_values(
-            by=indices,
-            inplace=True,
-            ascending=(str(order).lower() == 'asc')
-        )
-        _df.reset_index(drop=True, inplace=True)
+        if 'dask' not in _df.__module__:
+            _df.sort_values(
+                by=indices,
+                inplace=True,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df.reset_index(drop=True, inplace=True)
+        else:
+            _df = _df.sort_values(
+                by=indices,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df = _df.reset_index(drop=True)
         if limit is not None and len(_df) > limit:
             return _df.head(limit)
         return _df

meerschaum/core/Pipe/_sync.py

@@ -26,8 +26,6 @@ from meerschaum.utils.typing import (
     SuccessTuple,
     Dict,
     List,
-    Iterable,
-    Generator,
 )
 from meerschaum.utils.warnings import warn, error
 

meerschaum/jobs/__init__.py

@@ -11,11 +11,12 @@ import pathlib
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
-from meerschaum.jobs._Job import Job
+from meerschaum.jobs._Job import Job, StopMonitoringLogs
 from meerschaum.jobs._Executor import Executor
 
 __all__ = (
     'Job',
+    'StopMonitoringLogs',
     'systemd',
     'get_jobs',
     'get_filtered_jobs',