meerschaum 2.0.0rc6__py3-none-any.whl → 2.0.0rc8__py3-none-any.whl

meerschaum/config/_default.py

@@ -12,61 +12,61 @@ from meerschaum.connectors import attributes as connector_attributes
 from meerschaum.config._paths import SQLITE_DB_PATH
 
 default_meerschaum_config = {
-    'instance' : 'sql:main',
-    'api_instance' : 'MRSM{meerschaum:instance}',
-    'web_instance' : 'MRSM{meerschaum:instance}',
-    'default_repository' : 'api:mrsm',
-    'connectors' : {
-        'sql' : {
-            'default'      : {},
-            'main'         : {
-                'username' : 'mrsm',
-                'password' : 'mrsm',
-                'flavor'   : 'timescaledb',
-                'host'     : 'localhost',
-                'database' : 'meerschaum',
-                'port'     : 5432,
+    'instance': 'sql:main',
+    'api_instance': 'MRSM{meerschaum:instance}',
+    'web_instance': 'MRSM{meerschaum:instance}',
+    'default_repository': 'api:mrsm',
+    'connectors': {
+        'sql': {
+            'default': {},
+            'main': {
+                'username': 'mrsm',
+                'password': 'mrsm',
+                'flavor': 'timescaledb',
+                'host': 'localhost',
+                'database': 'meerschaum',
+                'port': 5432,
             },
-            'local'        : {
-                'flavor'   : 'sqlite',
-                'database' : str(SQLITE_DB_PATH),
+            'local': {
+                'flavor': 'sqlite',
+                'database': str(SQLITE_DB_PATH),
             },
             'memory': {
-                'flavor'   : 'sqlite',
-                'database' : ':memory:',
+                'flavor': 'sqlite',
+                'database': ':memory:',
             },
         },
-        'api' : {
-            'default'      : connector_attributes['api']['default'],
-            'main'         : {
-                'host'     : 'localhost',
-                'port'     : 8000,
+        'api': {
+            'default': connector_attributes['api']['default'],
+            'main': {
+                'host': 'localhost',
+                'port': 8000,
             },
-            'local'        : {
-                'host'     : 'localhost',
+            'local': {
+                'host': 'localhost',
             },
-            'mrsm'         : {
-                'host'     : 'api.mrsm.io',
-                'port'     : 443,
-                'protocol' : 'https',
+            'mrsm': {
+                'host': 'api.mrsm.io',
+                'port': 443,
+                'protocol': 'https',
             },
         },
     },
 }
 default_system_config = {
-    'connectors' : {
-        'all' : {
-            'pandas'           : 'pandas',
+    'connectors': {
+        'all': {
+            'pandas': 'pandas',
         },
-        'sql' : {
-            'chunksize'        : 100000,
-            'poolclass'        : 'sqlalchemy.pool.QueuePool',
-            'create_engine'    : {
-                'method'       : 'multi',
-                'pool_size'    : 5,
-                'max_overflow' : 10,
-                'pool_recycle' : 3600,
-                'connect_args' : {},
+        'sql': {
+            'chunksize': 100_000,
+            'poolclass': 'sqlalchemy.pool.QueuePool',
+            'create_engine': {
+                'method': 'multi',
+                'pool_size': 5,
+                'max_overflow': 10,
+                'pool_recycle': 3600,
+                'connect_args': {},
             },
         },
 
@@ -75,28 +75,28 @@ default_system_config = {
     },
     ### not to be confused with system_config['connectors']['api'], this is the configuration
     ### for the API server itself.
-    'api' : {
-        'uvicorn'          : {
-            'app'          : 'meerschaum.api:app',
-            'port'         : default_meerschaum_config['connectors']['api']['default']['port'],
-            'host'         : '0.0.0.0',
-            'workers'      : max(int(multiprocessing.cpu_count() / 2), 1),
+    'api': {
+        'uvicorn': {
+            'app': 'meerschaum.api:app',
+            'port': default_meerschaum_config['connectors']['api']['default']['port'],
+            'host': '0.0.0.0',
+            'workers': max(int(multiprocessing.cpu_count() / 2), 1),
         },
         'permissions':       {
-            'registration' : {
-                'users'    : True,
-                'pipes'    : True,
-                'plugins'  : True,
+            'registration': {
+                'users': True,
+                'pipes': True,
+                'plugins': True,
             },
-            'actions'      : {
+            'actions': {
                 'non_admin': True,
             },
             'chaining' : {
-                'insecure_parent_instance' : False,
-                'child_apis' : False,
+                'insecure_parent_instance': False,
+                'child_apis': False,
             },
         },
-        'protocol'         : default_meerschaum_config['connectors']['api']['default']['protocol'],
+        'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
     },
     'experimental': {
         'fetch': False,
@@ -106,21 +106,27 @@ default_system_config = {
         'inplace_sync': True,
     },
 }
-default_pipes_config       = {
-    'parameters'           : {
-        'columns'          : {
-            'datetime'     : None,
-            'id'           : None,
+default_pipes_config = {
+    'parameters': {
+        'columns': {
+            'datetime': None,
+            'id': None,
+        },
+        'chunk_minutes': 1440,
+        'fetch': {
+            'backtrack_minutes': 1440,
+        },
+        'verify': {
+            'bound_days': 366,
         },
-        'chunk_minutes'    : 1440,
     },
-    'attributes'           : {
+    'attributes': {
         'local_cache_timeout_seconds': 60,
     },
 }
-default_plugins_config     = {}
+default_plugins_config = {}
 default_experimental_config = {
-    'venv' : True,
+    'venv': True,
 }
 
 

meerschaum/config/_formatting.py

@@ -115,6 +115,32 @@ default_formatting_config = {
             },
         },
     },
+    'success_calm'         : {
+        'unicode'          : {
+            'icon'         : 'MRSM{formatting:emoji:success_calm}',
+        },
+        'ascii'            : {
+            'icon'         : '+',
+        },
+        'ansi'             : {
+            'rich'         : {
+                'style'    : 'pale_green3',
+            },
+        },
+    },
+    'failure_calm'         : {
+        'unicode'          : {
+            'icon'         : 'MRSM{formatting:emoji:failure_calm}',
+        },
+        'ascii'            : {
+            'icon'         : 'x',
+        },
+        'ansi'             : {
+            'rich'         : {
+                'style'    : 'indian red',
+            },
+        },
+    },
     'errors'               : {
         'unicode'          : {
             'icon'         : 'MRSM{formatting:emoji:error}',

meerschaum/config/_jobs.py

@@ -7,6 +7,8 @@ Default configuration for jobs.
 """
 
 default_jobs_config = {
+    'timeout_seconds': 8,
+    'check_timeout_interval_seconds': 0.1,
     'logs' : {
         'num_files_to_keep': 5,
         'max_file_size': 100_000,
@@ -14,11 +16,32 @@ default_jobs_config = {
         'refresh_files_seconds': 5.0,
         'min_buffer_len': 15,
         'colors' : [
-            'cyan', 'magenta', 'orange3', 'green', 'blue', 'red', 'spring_green3',
-            'medium_purple3', 'medium_violet_red', 'slate_blue1', 'bright_red', 'steel_blue3',
-            'aquamarine1', 'dark_khaki', 'pink3', 'gold3', 'pale_green1', 'light coral',
-            'light_goldenrod2', 'cornsilk1', 'orange_red1', 'deep_pink1', 'aquamarine3',
-            'sky_blue2', 'tan', 'honeydew2',
+            'cyan',
+            'magenta',
+            'orange3',
+            'green',
+            'blue',
+            'red',
+            'spring_green3',
+            'medium_purple3',
+            'medium_violet_red',
+            'slate_blue1', 
+            'bright_red',
+            'steel_blue3',
+            'aquamarine1',
+            'dark_khaki',
+            'pink3',
+            'gold3',
+            'pale_green1',
+            'light coral',
+            'light_goldenrod2',
+            'cornsilk1',
+            'orange_red1',
+            'deep_pink1',
+            'aquamarine3',
+            'sky_blue2',
+            'tan',
+            'honeydew2',
         ],
     },
 }

meerschaum/config/_paths.py

@@ -48,22 +48,38 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
                 Path(path).resolve()
                 for path in json.loads(os.environ[ENVIRONMENT_PLUGINS_DIR])
             ] if os.environ[ENVIRONMENT_PLUGINS_DIR].lstrip().startswith('[')
-            else [Path(os.environ[ENVIRONMENT_PLUGINS_DIR]).resolve()]
+            else [
+                Path(path_str).resolve()
+                for path_str in os.environ[ENVIRONMENT_PLUGINS_DIR].split(':')
+                if path_str
+            ]
         )
     except Exception as e:
         PLUGINS_DIR_PATHS = []
 
     if not PLUGINS_DIR_PATHS:
         print(
-            "Invalid plugins directories set for " +
-            f"environment variable '{ENVIRONMENT_PLUGINS_DIR}'.\n" +
-            f"Please enter a valid path or JSON-encoded paths for {ENVIRONMENT_PLUGINS_DIR}.",
-            file = sys.stderr,
+            "Invalid plugins directories set for "
+            f"environment variable '{ENVIRONMENT_PLUGINS_DIR}'.\n\n"
+            f"Set this to a colon-separated path string:\n\n"
+            f"`export {ENVIRONMENT_PLUGINS_DIR}=./plugins:/another/path/to/plugins`\n\n"
+            "or a JSON-encoded path list:\n\n"
+            f"`export {ENVIRONMENT_PLUGINS_DIR}=" + "'[\"./plugins\", \"/another/path/to/plugins\"]'`"
+            f"",
         )
         sys.exit(1)
 else:
     PLUGINS_DIR_PATHS = [_ROOT_DIR_PATH / 'plugins']
 
+### Remove duplicate plugins paths.
+_seen_plugins_paths, _plugins_paths_to_remove = set(), set()
+for _plugin_path in PLUGINS_DIR_PATHS:
+    if _plugin_path in _seen_plugins_paths:
+        _plugins_paths_to_remove.add(_plugin_path)
+    _seen_plugins_paths.add(_plugin_path)
+for _plugin_path in _plugins_paths_to_remove:
+    PLUGINS_DIR_PATHS.remove(_plugin_path)
+
 
 paths = {
     'PACKAGE_ROOT_PATH'              : str(Path(__file__).parent.parent.resolve()),

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.0.0rc6"
+__version__ = "2.0.0rc8"

meerschaum/connectors/api/_fetch.py

@@ -42,7 +42,7 @@ def fetch(
     remote_metric_key = instructions.get('metric_key', None)
     remote_location_key = instructions.get('location_key', None)
     if begin is None:
-        begin = pipe.sync_time
+        begin = pipe.get_sync_time(debug=debug)
 
     _params = copy.deepcopy(params) if params is not None else {}
     _params = apply_patch_to_config(_params, instructions.get('params', {}))

meerschaum/connectors/api/_pipes.py

@@ -7,6 +7,9 @@ Register or fetch Pipes from the API
 """
 
 from __future__ import annotations
+import time
+import json
+from io import StringIO
 from datetime import datetime
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
@@ -123,7 +126,6 @@ def fetch_pipes_keys(
     A list of tuples containing pipes' keys.
     """
     from meerschaum.config.static import STATIC_CONFIG
-    import json
     if connector_keys is None:
         connector_keys = []
     if metric_keys is None:
@@ -169,7 +171,6 @@ def sync_pipe(
     from meerschaum.utils.misc import json_serialize_datetime
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    import json, time
     begin = time.time()
     more_itertools = attempt_import('more_itertools')
     if df is None:
@@ -310,7 +311,6 @@ def get_pipe_data(
         **kw: Any
     ) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
-    import json
     r_url = pipe_r_url(pipe)
     chunks_list = []
     while True:
@@ -340,7 +340,7 @@ def get_pipe_data(
     from meerschaum.utils.dataframe import parse_df_datetimes
     pd = import_pandas()
     try:
-        df = pd.read_json(response.text)
+        df = pd.read_json(StringIO(response.text))
     except Exception as e:
         warn(f"Failed to parse response for {pipe}:\n{e}")
         return None
@@ -367,7 +367,6 @@ def get_backtrack_data(
         **kw: Any,
     ) -> pandas.DataFrame:
     """Get a Pipe's backtrack data from the API."""
-    import json
     r_url = pipe_r_url(pipe)
     try:
         response = self.get(
@@ -389,12 +388,12 @@ def get_backtrack_data(
         dprint(response.text)
     pd = import_pandas()
     try:
-        df = pd.read_json(response.text)
+        df = pd.read_json(StringIO(response.text))
     except Exception as e:
         warn(f"Failed to read response into a dataframe:\n{e}")
         return None
 
-    df = parse_df_datetimes(pd.read_json(response.text), debug=debug)
+    df = parse_df_datetimes(pd.read_json(StringIO(response.text)), debug=debug)
     return df
 
 def get_pipe_id(
@@ -438,7 +437,6 @@ def get_pipe_attributes(
     """
     r_url = pipe_r_url(pipe)
     response = self.get(r_url + '/attributes', debug=debug)
-    import json
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -474,7 +472,6 @@ def get_sync_time(
     """
     from meerschaum.utils.misc import is_int
     from meerschaum.utils.warnings import warn
-    import datetime, json
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
@@ -545,7 +542,6 @@ def create_metadata(
     """
     from meerschaum.utils.debug import dprint
     from meerschaum.config.static import STATIC_CONFIG
-    import json
     r_url = STATIC_CONFIG['api']['endpoints']['metadata']
     response = self.post(r_url, debug=debug)
     if debug:
@@ -590,7 +586,6 @@ def get_pipe_rowcount(
     The number of rows in the pipe's table, bound the given parameters.
     If the table does not exist, return 0.
     """
-    import json
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + "/rowcount",

meerschaum/connectors/sql/_fetch.py

@@ -148,7 +148,7 @@ def get_pipe_metadef(
         dt_name = sql_item_name(_dt, self.flavor)
         is_guess = False
 
-    if begin is not None or end is not None:
+    if begin not in (None, '') or end is not None:
         if is_guess:
             if _dt is None:
                 warn(
@@ -168,20 +168,38 @@ def get_pipe_metadef(
     if 'order by' in definition.lower() and 'over' not in definition.lower():
         error("Cannot fetch with an ORDER clause in the definition")
 
+    apply_backtrack = begin == ''
     begin = (
-        begin if not (isinstance(begin, str) and begin == '')
-        else pipe.get_sync_time(debug=debug)
+        pipe.get_sync_time(debug=debug)
+        if begin == ''
+        else begin
     )
-        
+
+    if begin and end and begin >= end:
+        begin = None
+    
     da = None
     if dt_name:
-        ### default: do not backtrack
-        begin_da = dateadd_str(
-            flavor=self.flavor, datepart='minute', number=(-1 * btm), begin=begin,
-        ) if begin else None
-        end_da = dateadd_str(
-            flavor=self.flavor, datepart='minute', number=1, begin=end,
-        ) if end else None
+        begin_da = (
+            dateadd_str(
+                flavor = self.flavor,
+                datepart = 'minute',
+                number = ((-1 * btm) if apply_backtrack else 0), 
+                begin = begin,
+            )
+            if begin
+            else None
+        )
+        end_da = (
+            dateadd_str(
+                flavor = self.flavor,
+                datepart = 'minute',
+                number = 0,
+                begin = end,
+            )
+            if end
+            else None
+        )
 
     meta_def = (
         _simple_fetch_query(pipe) if (

meerschaum/connectors/sql/_pipes.py

@@ -6,7 +6,7 @@
 Interact with Pipes metadata via SQLConnector.
 """
 from __future__ import annotations
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
 import meerschaum as mrsm
 from meerschaum.utils.typing import (
     Union, Any, SuccessTuple, Tuple, Dict, Optional, List
@@ -427,10 +427,17 @@ def get_create_index_queries(
                 get_distinct_col_count(_id, f"SELECT {_id_name} FROM {_pipe_name}", self)
                 if (_id is not None and _create_space_partition) else None
             )
+
+            chunk_interval = pipe.get_chunk_interval(debug=debug)
+            chunk_interval_minutes = (
+                chunk_interval
+                if isinstance(chunk_interval, int)
+                else int(chunk_interval.total_seconds() / 60)
+            )
             chunk_time_interval = (
-                pipe.parameters.get('chunk_time_interval', None)
-                or
-                ("INTERVAL '1 DAY'" if not 'int' in _datetime_type.lower() else '100000')
+                f"INTERVAL '{chunk_interval_minutes} MINUTES'"
+                if isinstance(chunk_interval, timedelta)
+                else f'{chunk_interval_minutes}'
             )
 
             dt_query = (

meerschaum/connectors/sql/_sql.py

@@ -102,10 +102,6 @@ def read(
         `chunksize` must not be `None` (falls back to 1000 if so),
         and hooks are not called in this case.
 
-    as_dask: bool, default False
-        If `True`, return a `dask.DataFrame`
-        (which may be loaded into a Pandas DataFrame with `df.compute()`).
-
     index_col: Optional[str], default None
         If using Dask, use this column as the index column.
         If omitted, a Pandas DataFrame will be fetched and converted to a Dask DataFrame.
@@ -134,7 +130,6 @@ def read(
     is_dask = 'dask' in pd.__name__
     pd = attempt_import('pandas')
     #  pd = import_pandas()
-    dd = attempt_import('dask.dataframe') if as_dask else None
     is_dask = dd is not None
     npartitions = chunksize_to_npartitions(chunksize)
     if is_dask:
@@ -687,7 +682,7 @@ def to_sql(
     from meerschaum.utils.packages import attempt_import, import_pandas
     sqlalchemy = attempt_import('sqlalchemy', debug=debug)
     pd = import_pandas()
-    is_dask = 'dask' in pd.__name__
+    is_dask = 'dask' in df.__module__
 
     stats = {'target': name, }
     ### resort to defaults if None

meerschaum/core/Pipe/__init__.py

@@ -119,7 +119,11 @@ class Pipe:
         _get_chunk_label,
         get_num_workers,
     )
-    from ._verify import verify
+    from ._verify import (
+        verify,
+        get_bound_interval,
+        get_bound_time,
+    )
     from ._delete import delete
     from ._drop import drop
     from ._clear import clear

meerschaum/core/Pipe/_data.py

@@ -20,6 +20,7 @@ def get_data(
         params: Optional[Dict[str, Any]] = None,
         as_iterator: bool = False,
         as_chunks: bool = False,
+        as_dask: bool = False,
         chunk_interval: Union[timedelta, int, None] = None,
         fresh: bool = False,
         debug: bool = False,
@@ -57,6 +58,10 @@ def get_data(
     as_chunks: bool, default False
         Alias for `as_iterator`.
 
+    as_dask: bool, default False
+        If `True`, return a `dask.DataFrame`
+        (which may be loaded into a Pandas DataFrame with `df.compute()`).
+
     chunk_interval: Union[timedelta, int, None], default None
         If `as_iterator`, then return chunks with `begin` and `end` separated by this interval.
         This may be set under `pipe.parameters['chunk_minutes']`.
@@ -85,6 +90,9 @@ def get_data(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import iterate_chunks, items_str
     from meerschaum.utils.dataframe import add_missing_cols_to_df
+    from meerschaum.utils.packages import attempt_import
+    dd = attempt_import('dask.dataframe') if as_dask else None
+    dask = attempt_import('dask') if as_dask else None
 
     if select_columns == '*':
         select_columns = None
@@ -108,6 +116,33 @@ def get_data(
             debug = debug,
         )
 
+    if as_dask:
+        from multiprocessing.pool import ThreadPool
+        dask_pool = ThreadPool(self.get_num_workers())
+        dask.config.set(pool=dask_pool)
+        chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
+        bounds = self.get_chunk_bounds(
+            begin = begin,
+            end = end,
+            bounded = False,
+            chunk_interval = chunk_interval,
+            debug = debug,
+        )
+        dask_chunks = [
+            dask.delayed(self.get_data)(
+                select_columns = select_columns,
+                omit_columns = omit_columns,
+                begin = chunk_begin,
+                end = chunk_end,
+                params = params,
+                chunk_interval = chunk_interval,
+                fresh = fresh,
+                debug = debug,
+            )
+            for (chunk_begin, chunk_end) in bounds
+        ]
+        return dd.from_delayed(dask_chunks)
+
     if not self.exists(debug=debug):
         return None
        
@@ -245,12 +280,7 @@ def _get_data_as_iterator(
         elif isinstance(max_dt, datetime):
             max_dt = round_time(max_dt + timedelta(minutes=1))
 
-    if chunk_interval is None:
-        chunk_interval = self.get_chunk_interval(debug=debug)
-    elif isinstance(chunk_interval, int) and isinstance(min_dt, datetime):
-        chunk_interval = timedelta(minutes=1)
-    elif isinstance(chunk_interval, timedelta) and isinstance(min_dt, int):
-        chunk_interval = int(chunk_interval.total_seconds() / 60)
+    chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
 
     ### If we can't determine bounds
     ### or if chunk_interval exceeds the max,
@@ -458,14 +488,34 @@ def get_rowcount(
 
 def get_chunk_interval(
         self,
+        chunk_interval: Union[timedelta, int, None] = None,
         debug: bool = False,
     ) -> Union[timedelta, int]:
     """
     Get the chunk interval to use for this pipe.
+
+    Parameters
+    ----------
+    chunk_interval: Union[timedelta, int, None], default None
+        If provided, coerce this value into the correct type.
+        For example, if the datetime axis is an integer, then
+        return the number of minutes.
+
+    Returns
+    -------
+    The chunk interval (`timedelta` or `int`) to use with this pipe's `datetime` axis.
     """
     default_chunk_minutes = get_config('pipes', 'parameters', 'chunk_minutes')
     configured_chunk_minutes = self.parameters.get('chunk_minutes', None)
-    chunk_minutes = configured_chunk_minutes or default_chunk_minutes
+    chunk_minutes = (
+        (configured_chunk_minutes or default_chunk_minutes)
+        if chunk_interval is None
+        else (
+            chunk_interval
+            if isinstance(chunk_interval, int)
+            else int(chunk_interval.total_seconds() / 60)
+        )
+    )
 
     dt_col = self.columns.get('datetime', None)
     if dt_col is None:
@@ -529,8 +579,7 @@ def get_chunk_bounds(
         return [(None, None)]
 
     ### Set the chunk interval under `pipe.parameters['chunk_minutes']`.
-    if chunk_interval is None:
-        chunk_interval = self.get_chunk_interval(debug=debug)
+    chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
     ### Build a list of tuples containing the chunk boundaries
     ### so that we can sync multiple chunks in parallel.