meerschaum 2.4.0rc1__tar.gz → 2.4.1__tar.gz

{meerschaum-2.4.0rc1/meerschaum.egg-info → meerschaum-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.4.0rc1
+Version: 2.4.1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -61,6 +61,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "core"
 Requires-Dist: more-itertools>=8.7.0; extra == "core"
 Requires-Dist: fasteners>=0.19.0; extra == "core"
 Requires-Dist: virtualenv>=20.1.0; extra == "core"
+Requires-Dist: attrs<24.2.0; extra == "core"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "core"
 Requires-Dist: uv>=0.2.11; extra == "core"
 Provides-Extra: jobs
@@ -156,6 +157,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "sql"
 Requires-Dist: more-itertools>=8.7.0; extra == "sql"
 Requires-Dist: fasteners>=0.19.0; extra == "sql"
 Requires-Dist: virtualenv>=20.1.0; extra == "sql"
+Requires-Dist: attrs<24.2.0; extra == "sql"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "sql"
 Requires-Dist: uv>=0.2.11; extra == "sql"
 Provides-Extra: dash
@@ -212,6 +214,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "api"
 Requires-Dist: more-itertools>=8.7.0; extra == "api"
 Requires-Dist: fasteners>=0.19.0; extra == "api"
 Requires-Dist: virtualenv>=20.1.0; extra == "api"
+Requires-Dist: attrs<24.2.0; extra == "api"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "api"
 Requires-Dist: uv>=0.2.11; extra == "api"
 Requires-Dist: pprintpp>=0.4.0; extra == "api"
@@ -261,6 +264,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "full"
 Requires-Dist: more-itertools>=8.7.0; extra == "full"
 Requires-Dist: fasteners>=0.19.0; extra == "full"
 Requires-Dist: virtualenv>=20.1.0; extra == "full"
+Requires-Dist: attrs<24.2.0; extra == "full"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "full"
 Requires-Dist: uv>=0.2.11; extra == "full"
 Requires-Dist: dill>=0.3.3; extra == "full"

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/api/dash/callbacks/dashboard.py

@@ -214,6 +214,13 @@ def update_content(*args):
     trigger = ctx.triggered[0]['prop_id'].split('.')[0] if not trigger else trigger
 
     session_data = args[-1]
+    mrsm_location_href = session_data.get('mrsm-location.href', None)
+    if (
+        initial_load
+        and mrsm_location_href
+        and not mrsm_location_href.rstrip('/').endswith('/dash')
+    ):
+        raise PreventUpdate
 
     ### NOTE: functions MUST return a list of content and a list of alerts
     triggers = {

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/api/dash/callbacks/pipes.py

@@ -5,7 +5,9 @@
 Define callbacks for the `/dash/pipes/` page.
 """
 
-from dash.dependencies import Input, Output
+from urllib.parse import parse_qs
+
+from dash.dependencies import Input, Output, State
 from dash import no_update
 
 import meerschaum as mrsm
@@ -13,17 +15,28 @@ from meerschaum.api.dash import dash_app
 from meerschaum.api.dash.pipes import build_pipe_card
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.utils.packages import import_html, import_dcc
+from meerschaum.api.dash.sessions import is_session_authenticated
+from meerschaum.utils.typing import Optional, Dict, Any
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 
 
 @dash_app.callback(
     Output('pipe-output-div', 'children'),
     Input('pipes-location', 'pathname'),
+    State('pipes-location', 'search'),
+    State('session-store', 'data'),
 )
-def render_page_from_url(pathname):
+def render_page_from_url(
+    pathname: str,
+    pipe_search: str,
+    session_data: Optional[Dict[str, Any]],
+):
     if not str(pathname).startswith('/dash/pipes'):
         return no_update
 
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+
     keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
     if len(keys) not in (2, 3):
         return no_update
@@ -31,11 +44,12 @@ def render_page_from_url(pathname):
     ck = keys[0]
     mk = keys[1]
     lk = keys[2] if len(keys) == 3 else None
+    query_params = parse_qs(pipe_search.lstrip('?')) if pipe_search else {}
+    instance = query_params.get('instance', [None])[0]
 
-    pipe = mrsm.Pipe(ck, mk, lk)
-    ### TODO Check if logged in
+    pipe = mrsm.Pipe(ck, mk, lk, instance=instance)
     return [
         html.Br(),
-        build_pipe_card(pipe, authenticated=False),
+        build_pipe_card(pipe, authenticated=authenticated, include_manage=False),
         html.Br(),
     ]

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/api/dash/pipes.py

@@ -10,6 +10,8 @@ from __future__ import annotations
 import json
 import shlex
 from textwrap import dedent
+from urllib.parse import urlencode
+
 from dash.dependencies import Input, Output, State
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
 from meerschaum.utils.misc import string_to_dict
@@ -30,6 +32,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 humanfriendly = attempt_import('humanfriendly', check_update=CHECK_UPDATE)
 pd = import_pandas()
 
+
 def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, None]:
     """
     Return a `meerschaum.Pipe` object from a dynamic object with an
@@ -105,6 +108,7 @@ def pipes_from_state(
 def build_pipe_card(
     pipe: mrsm.Pipe,
     authenticated: bool = False,
+    include_manage: bool = True,
     _build_children_num: int = 10,
 ) -> 'dbc.Card':
     """
@@ -118,6 +122,9 @@ def build_pipe_card(
     authenticated: bool, default False
         If `True`, allow editing functionality to the card.
 
+    include_manage: bool, default True
+        If `True` and `authenticated` is `True`, include the "Manage" dropdown.
+
     Returns
     -------
     A dash bootstrap components Card representation of the pipe.
@@ -184,7 +191,7 @@ def build_pipe_card(
                         size='sm',
                         color='secondary',
                     )
-                ) if authenticated else [],
+                ) if authenticated and include_manage else [],
                 width=2,
             ),
             dbc.Col(width=6),
@@ -217,8 +224,16 @@ def build_pipe_card(
 
     ]
 
+    query_params = {}
+    default_instance = get_config('meerschaum', 'instance')
+    if pipe.instance_keys != default_instance:
+        query_params['instance'] = pipe.instance_keys
     pipe_url = (
-        f"/dash/pipes/{pipe.connector_keys}/{pipe.metric_key}/{pipe.location_key}"
+        f"/dash/pipes/"
+        + f"{pipe.connector_keys}/"
+        + f"{pipe.metric_key}/"
+        + (f"{pipe.location_key}" if pipe.location_key is not None else '')
+        + (f"?{urlencode(query_params)}" if query_params else "")
     )
 
     card_header_children = dbc.Row(

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.0rc1"
+__version__ = "2.4.1"

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/connectors/sql/_pipes.py

@@ -909,7 +909,7 @@ def get_pipe_data_query(
 
     cols_names = [sql_item_name(col, self.flavor, None) for col in select_columns]
     select_cols_str = (
-        'SELECT\n'
+        'SELECT\n    '
         + ',\n    '.join(
             [
                 (
@@ -953,14 +953,14 @@ def get_pipe_data_query(
                 warn(
                     f"No datetime could be determined for {pipe}."
                     + "\n    Ignoring begin and end...",
-                    stack = False,
+                    stack=False,
                 )
                 begin, end = None, None
             else:
                 warn(
                     f"A datetime wasn't specified for {pipe}.\n"
                     + f"    Using column \"{_dt}\" for datetime bounds...",
-                    stack = False,
+                    stack=False,
                 )
 
     is_dt_bound = False
@@ -1014,9 +1014,12 @@ def get_pipe_data_query(
 
     if isinstance(limit, int):
         if self.flavor == 'mssql':
-            query = f'SELECT TOP {limit} ' + query[len("SELECT *"):]
+            query = f'SELECT TOP {limit}\n' + query[len("SELECT "):]
         elif self.flavor == 'oracle':
-            query = f"SELECT * FROM (\n  {query}\n)\nWHERE ROWNUM = 1"
+            query = (
+                f"SELECT * FROM (\n  {query}\n)\n"
+                + f"WHERE ROWNUM IN ({', '.join([str(i) for i in range(1, limit+1)])})"
+            )
         else:
             query += f"\nLIMIT {limit}"
     

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/core/Pipe/_data.py

@@ -106,7 +106,7 @@ def get_data(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import iterate_chunks, items_str
     from meerschaum.utils.dtypes import to_pandas_dtype
-    from meerschaum.utils.dataframe import add_missing_cols_to_df
+    from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
@@ -122,20 +122,29 @@ def get_data(
     as_iterator = as_iterator or as_chunks
 
     def _sort_df(_df):
+        if df_is_chunk_generator(_df):
+            return _df
         dt_col = self.columns.get('datetime', None)
         indices = [] if dt_col not in _df.columns else [dt_col]
         non_dt_cols = [
             col
-            for col_ix, col in self.columns.values()
+            for col_ix, col in self.columns.items()
             if col_ix != 'datetime' and col in _df.columns
         ]
         indices.extend(non_dt_cols)
-        _df.sort_values(
-            by=indices,
-            inplace=True,
-            ascending=(str(order).lower() == 'asc')
-        )
-        _df.reset_index(drop=True, inplace=True)
+        if 'dask' not in _df.__module__:
+            _df.sort_values(
+                by=indices,
+                inplace=True,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df.reset_index(drop=True, inplace=True)
+        else:
+            _df = _df.sort_values(
+                by=indices,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df = _df.reset_index(drop=True)
         if limit is not None and len(_df) > limit:
             return _df.head(limit)
         return _df

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/core/Pipe/_sync.py

@@ -26,8 +26,6 @@ from meerschaum.utils.typing import (
     SuccessTuple,
     Dict,
     List,
-    Iterable,
-    Generator,
 )
 from meerschaum.utils.warnings import warn, error
 

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/jobs/__init__.py

@@ -11,11 +11,12 @@ import pathlib
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
-from meerschaum.jobs._Job import Job
+from meerschaum.jobs._Job import Job, StopMonitoringLogs
 from meerschaum.jobs._Executor import Executor
 
 __all__ = (
     'Job',
+    'StopMonitoringLogs',
     'systemd',
     'get_jobs',
     'get_filtered_jobs',

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/utils/packages/_packages.py

@@ -49,6 +49,7 @@ packages: Dict[str, Dict[str, str]] = {
         'more_itertools'             : 'more-itertools>=8.7.0',
         'fasteners'                  : 'fasteners>=0.19.0',
         'virtualenv'                 : 'virtualenv>=20.1.0',
+        'attrs'                      : 'attrs<24.2.0',
         'apscheduler'                : 'APScheduler>=4.0.0a5',
         'uv'                         : 'uv>=0.2.11',
     },

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum/utils/schedule.py

@@ -101,6 +101,7 @@ def schedule_function(
     kw['debug'] = debug
     kw = filter_keywords(function, **kw)
 
+    _ = mrsm.attempt_import('attrs', lazy=False)
     apscheduler = mrsm.attempt_import('apscheduler', lazy=False)
     now = round_time(datetime.now(timezone.utc), timedelta(minutes=1))
     trigger = parse_schedule(schedule, now=now)

{meerschaum-2.4.0rc1 → meerschaum-2.4.1/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.4.0rc1
+Version: 2.4.1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -61,6 +61,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "core"
 Requires-Dist: more-itertools>=8.7.0; extra == "core"
 Requires-Dist: fasteners>=0.19.0; extra == "core"
 Requires-Dist: virtualenv>=20.1.0; extra == "core"
+Requires-Dist: attrs<24.2.0; extra == "core"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "core"
 Requires-Dist: uv>=0.2.11; extra == "core"
 Provides-Extra: jobs
@@ -156,6 +157,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "sql"
 Requires-Dist: more-itertools>=8.7.0; extra == "sql"
 Requires-Dist: fasteners>=0.19.0; extra == "sql"
 Requires-Dist: virtualenv>=20.1.0; extra == "sql"
+Requires-Dist: attrs<24.2.0; extra == "sql"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "sql"
 Requires-Dist: uv>=0.2.11; extra == "sql"
 Provides-Extra: dash
@@ -212,6 +214,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "api"
 Requires-Dist: more-itertools>=8.7.0; extra == "api"
 Requires-Dist: fasteners>=0.19.0; extra == "api"
 Requires-Dist: virtualenv>=20.1.0; extra == "api"
+Requires-Dist: attrs<24.2.0; extra == "api"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "api"
 Requires-Dist: uv>=0.2.11; extra == "api"
 Requires-Dist: pprintpp>=0.4.0; extra == "api"
@@ -261,6 +264,7 @@ Requires-Dist: prompt-toolkit>=3.0.39; extra == "full"
 Requires-Dist: more-itertools>=8.7.0; extra == "full"
 Requires-Dist: fasteners>=0.19.0; extra == "full"
 Requires-Dist: virtualenv>=20.1.0; extra == "full"
+Requires-Dist: attrs<24.2.0; extra == "full"
 Requires-Dist: APScheduler>=4.0.0a5; extra == "full"
 Requires-Dist: uv>=0.2.11; extra == "full"
 Requires-Dist: dill>=0.3.3; extra == "full"

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum.egg-info/SOURCES.txt

@@ -265,6 +265,7 @@ tests/test_actions.py
 tests/test_arguments.py
 tests/test_deduplicate.py
 tests/test_jobs.py
+tests/test_pipe_data.py
 tests/test_pipes_dtypes.py
 tests/test_sql.py
 tests/test_sync.py

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/meerschaum.egg-info/requires.txt

@@ -44,6 +44,7 @@ prompt-toolkit>=3.0.39
 more-itertools>=8.7.0
 fasteners>=0.19.0
 virtualenv>=20.1.0
+attrs<24.2.0
 APScheduler>=4.0.0a5
 uv>=0.2.11
 pprintpp>=0.4.0
@@ -97,6 +98,7 @@ prompt-toolkit>=3.0.39
 more-itertools>=8.7.0
 fasteners>=0.19.0
 virtualenv>=20.1.0
+attrs<24.2.0
 APScheduler>=4.0.0a5
 uv>=0.2.11
 
@@ -188,6 +190,7 @@ prompt-toolkit>=3.0.39
 more-itertools>=8.7.0
 fasteners>=0.19.0
 virtualenv>=20.1.0
+attrs<24.2.0
 APScheduler>=4.0.0a5
 uv>=0.2.11
 dill>=0.3.3
@@ -285,6 +288,7 @@ prompt-toolkit>=3.0.39
 more-itertools>=8.7.0
 fasteners>=0.19.0
 virtualenv>=20.1.0
+attrs<24.2.0
 APScheduler>=4.0.0a5
 uv>=0.2.11
 

meerschaum-2.4.1/tests/test_pipe_data.py

@@ -0,0 +1,89 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Tests for reading data from a pipe.
+"""
+
+import pytest
+from meerschaum import Pipe
+
+from tests.connectors import conns, get_flavors
+from tests import debug
+
+
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_get_backtrack_data_limit(flavor: str):
+    """
+    Test reading backtrack data
+    """
+    conn = conns[flavor]
+    pipe = Pipe('test', 'get_backtrack_data_limit', 'foo', instance=conn)
+    _ = pipe.delete()
+    pipe = Pipe(
+        'test', 'get_backtrack_data_limit', 'foo',
+        columns={'datetime': 'id'},
+        dtypes={'id': 'int'},
+        instance=conn,
+    )
+    success, msg = pipe.sync(
+        [
+            {'id': 1, 'color': 'red'},
+            {'id': 2, 'color': 'green'},
+            {'id': 3, 'color': 'blue'},
+        ],
+        check_existing=False,
+        debug=debug,
+    )
+    assert success, msg
+
+    limit = 1
+    df = pipe.get_backtrack_data(
+        limit=limit,
+        debug=debug,
+    )
+    assert df['id'][0] == 3
+    assert len(df) == limit
+
+
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_get_data_order(flavor: str):
+    """
+    Test reading backtrack data
+    """
+    conn = conns[flavor]
+    pipe = Pipe('test', 'get_data_order', 'foo', instance=conn)
+    _ = pipe.delete()
+    pipe = Pipe(
+        'test', 'get_data_order', 'foo',
+        columns={'datetime': 'id'},
+        dtypes={'id': 'int'},
+        instance=conn,
+    )
+    success, msg = pipe.sync(
+        [
+            {'id': 1, 'color': 'red'},
+            {'id': 2, 'color': 'green'},
+            {'id': 3, 'color': 'blue'},
+        ],
+        check_existing=False,
+        debug=debug,
+    )
+    assert success, msg
+
+    limit = 2
+    df = pipe.get_data(
+        limit=limit,
+        order='asc',
+        debug=debug,
+    )
+    assert len(df) == limit
+    assert df['id'][0] == 1
+
+    df = pipe.get_data(
+        limit=limit,
+        order='desc',
+        debug=debug,
+    )
+    assert len(df) == limit
+    assert df['id'][0] == 3

{meerschaum-2.4.0rc1 → meerschaum-2.4.1}/tests/test_pipes_dtypes.py

@@ -10,7 +10,6 @@ from tests.connectors import conns, get_flavors
 from tests.test_users import test_register_user
 import meerschaum as mrsm
 from meerschaum import Pipe
-from meerschaum.actions import actions
 from meerschaum.utils.dtypes import are_dtypes_equal
 from meerschaum.utils.sql import sql_item_name