meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -226,7 +226,7 @@ def parse_line(line: str) -> Dict[str, Any]:
     import shlex
     try:
         return parse_arguments(shlex.split(line))
-    except Exception as e:
+    except Exception:
         return {'action': [], 'text': line}
 
 

meerschaum/actions/delete.py

@@ -9,6 +9,7 @@ Functions for deleting elements.
 from __future__ import annotations
 from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List
 
+
 def delete(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -21,7 +22,6 @@ def delete(
 
     """
     from meerschaum.actions import choose_subaction
-    from meerschaum.utils.debug import dprint
     options = {
         'config'     : _delete_config, 
         'pipes'      : _delete_pipes,
@@ -65,19 +65,18 @@ def _complete_delete(
 
 
 def _delete_pipes(
-        debug: bool = False,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    debug: bool = False,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipes and delete their registrations.
-
     """
     from meerschaum import get_pipes
     from meerschaum.utils.prompt import yes_no
-    from meerschaum.utils.formatting import pprint, highlight_pipes
+    from meerschaum.utils.formatting import highlight_pipes
     from meerschaum.utils.warnings import warn
     from meerschaum.actions import actions
     pipes = get_pipes(as_list=True, debug=debug, **kw)
@@ -126,21 +125,22 @@ def _delete_pipes(
     
     return successes > 0, msg
 
+
 def _delete_config(
-        action: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete configuration files.
-
     """
-    import os, shutil
+    import os
+    import shutil
     from meerschaum.utils.prompt import yes_no
-    from meerschaum.config._paths import CONFIG_DIR_PATH, STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
+    from meerschaum.config._paths import STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH
     from meerschaum.config._read_config import get_possible_keys, get_keyfile_path
     from meerschaum.utils.debug import dprint
     paths = [p for p in [STACK_COMPOSE_PATH, DEFAULT_CONFIG_DIR_PATH] if p.exists()]
@@ -157,7 +157,7 @@ def _delete_config(
     else:
         sep = '\n' + '  - '
         answer = yes_no(
-            f"Are you sure you want to delete the following configuration files?" +
+            "Are you sure you want to delete the following configuration files?" +
             f"{sep + sep.join([str(p) for p in paths])}\n",
             default='n', noask=noask, yes=yes
         )
@@ -176,18 +176,18 @@ def _delete_config(
     
     return success, msg
 
+
 def _delete_plugins(
-        action: Optional[List[str]] = None,
-        repository: Optional[str] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    repository: Optional[str] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete plugins from a Meerschaum repository.
-
     """
     from meerschaum.utils.warnings import info
     from meerschaum.plugins import reload_plugins
@@ -206,7 +206,7 @@ def _delete_plugins(
     ) if not force else True
 
     if not answer:
-        return False, f"No plugins deleted."
+        return False, "No plugins deleted."
 
     successes = {}
     for name in action:
@@ -218,27 +218,24 @@ def _delete_plugins(
     reload_plugins(debug=debug)
     return True, "Success"
 
+
 def _delete_users(
-        action: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        shell: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    shell: bool = False,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete users from a Meerschaum instance. Adequate permissions are required.
-
     """
-    from meerschaum import get_connector
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum.utils.prompt import yes_no, prompt
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum.utils.warnings import info
     from meerschaum.core import User
-    from meerschaum.connectors.api import APIConnector
     from meerschaum.utils.formatting import print_tuple
     instance_connector = parse_instance_keys(mrsm_instance)
 
@@ -292,24 +289,25 @@ def _delete_users(
     )
     return True, msg
 
+
 def _delete_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Delete configured connectors.
     
     Example:
         `delete connectors sql:test`
-
     """
-    import os, pathlib
-    from meerschaum.utils.prompt import yes_no, prompt
+    import os
+    import pathlib
+    from meerschaum.utils.prompt import yes_no
     from meerschaum.connectors.parse import parse_connector_keys
     from meerschaum.config import _config
     from meerschaum.config._edit import write_config
@@ -329,7 +327,7 @@ def _delete_connectors(
     for ck in _keys:
         try:
             conn = parse_connector_keys(ck, debug=debug)
-        except Exception as e:
+        except Exception:
             warn(f"Could not parse connector '{ck}'. Skipping...", stack=False)
             continue
 
@@ -357,26 +355,27 @@ def _delete_connectors(
                     ):
                         try:
                             os.remove(c.database)
-                        except Exception as e:
+                        except Exception:
                             warn(
                                 "Failed to delete database file for connector "
                                 + f"'{c}'. Ignoring...", stack=False
                             )
-        except Exception as e:
+        except Exception:
             pass
         try:
             del cf['meerschaum']['connectors'][c.type][c.label]
-        except Exception as e:
+        except Exception:
             warn(f"Failed to delete connector '{c}' from configuration. Skipping...", stack=False)
 
     write_config(cf, debug=debug)
     return True, "Success"
 
+
 def _complete_delete_connectors(
-        action: Optional[List[str]] = None,
-        line: str = '',
-        **kw: Any
-    ) -> List[str]:
+    action: Optional[List[str]] = None,
+    line: str = '',
+    **kw: Any
+) -> List[str]:
     from meerschaum.config import get_config
     from meerschaum.utils.misc import get_connector_labels
     types = list(get_config('meerschaum', 'connectors').keys())
@@ -401,10 +400,8 @@ def _delete_jobs(
     Remove a job's log files and delete the job's ID.
     
     If the job is running, ask to kill the job first.
-
     """
     from meerschaum.jobs import (
-        Job,
         get_running_jobs,
         get_stopped_jobs,
         get_filtered_jobs,
@@ -460,7 +457,7 @@ def _delete_jobs(
             ### Ensure the running jobs are dead.
             if get_running_jobs(executor_keys, jobs, debug=debug):
                 return False, (
-                    f"Failed to kill running jobs. Please stop these jobs before deleting."
+                    "Failed to kill running jobs. Please stop these jobs before deleting."
                 )
             _to_delete.update(to_stop_jobs)
 
@@ -475,7 +472,7 @@ def _delete_jobs(
         pprint_jobs(_to_delete, nopretty=nopretty)
         if not yes_no(
             "Are you sure you want to delete these jobs?",
-            yes=yes, noask=noask, default='y',
+            yes=yes, noask=noask, default='n',
         ):
             return False, "No jobs were deleted."
 
@@ -569,7 +566,6 @@ def _delete_venvs(
     from meerschaum.utils.venv import venv_exists
     from meerschaum.utils.prompt import yes_no
     from meerschaum.utils.misc import print_options
-    from meerschaum.utils.warnings import warn
 
     venvs_to_skip = ['mrsm']
     venvs = [

meerschaum/actions/edit.py

@@ -397,6 +397,7 @@ def _edit_jobs(
     from meerschaum._internal.arguments import (
         split_pipeline_sysargs,
         split_chained_sysargs,
+        parse_arguments,
     )
     from meerschaum.utils.formatting import make_header, print_options
     from meerschaum.utils.warnings import info
@@ -410,7 +411,26 @@ def _edit_jobs(
 
     num_edited = 0
     for name, job in jobs.items():
-        sysargs_str = shlex.join(job.sysargs)
+        try:
+            sub_args_line = None
+            pipeline_args_line = None
+            if job.sysargs[:2] == ['start', 'pipeline']:
+                job_args = parse_arguments(job.sysargs)
+                mrsm.pprint(job_args)
+                sub_args_line = job_args['params']['sub_args_line']
+                params_index = job.sysargs[2:].index('-P')
+                indices_to_skip = (params_index, params_index + 1)
+                pipeline_args_line = shlex.join(
+                    [a for i, a in enumerate(job.sysargs[2:]) if i not in indices_to_skip]
+                )
+        except (ValueError, IndexError):
+            sub_args_line = None
+
+        sysargs_str = (
+            f"{sub_args_line} : {pipeline_args_line}"
+            if sub_args_line is not None and pipeline_args_line is not None
+            else shlex.join(job.sysargs)
+        )
         clear_screen(debug=debug)
         info(
             f"Editing arguments for job '{name}'.\n"
@@ -422,7 +442,7 @@ def _edit_jobs(
         try:
             new_sysargs_str = prompt(
                 "",
-                default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
+                default_editable=job.label,
                 multiline=True,
                 icon=False,
                 completer=ShellCompleter(),

meerschaum/actions/install.py

@@ -175,7 +175,7 @@ def _install_packages(
             + f" into the virtual environment '{venv}'."
         )
     return False, (
-        f"Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
+        "Failed to install package" + ("s" if len(action) != 1 else '') + f" {items_str(action)}."
     )
 
 
@@ -200,7 +200,6 @@ def _install_required(
     from meerschaum.core import Plugin
     from meerschaum.utils.warnings import warn, info
     from meerschaum.connectors.parse import parse_repo_keys
-    from meerschaum.utils.formatting import print_tuple
     from meerschaum.plugins import get_plugins_names
     repo_connector = parse_repo_keys(repository)
 

meerschaum/actions/sync.py

@@ -282,9 +282,8 @@ def _sync_pipes(
     from meerschaum.utils.formatting import print_pipes_results
     from meerschaum.config.static import STATIC_CONFIG
 
-    ### NOTE: Removed MRSM_NONINTERACTIVE check.
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
-    _ = noninteractive_val in ('1', 'true', 'True', 'yes')
+    noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
 
     run = True
     msg = ""
@@ -292,7 +291,7 @@ def _sync_pipes(
     cooldown = 2 * (min_seconds + 1)
     success_pipes, failure_pipes = [], []
     while run:
-        _progress = progress() if shell else None
+        _progress = progress() if shell and not noninteractive else None
         cm = _progress if _progress is not None else contextlib.nullcontext()
 
         lap_begin = time.perf_counter()

meerschaum/api/routes/_pipes.py

@@ -11,7 +11,6 @@ from __future__ import annotations
 
 import io
 import json
-import fastapi
 from decimal import Decimal
 import datetime
 
@@ -359,16 +358,16 @@ def sync_pipe(
     p = get_pipe(connector_keys, metric_key, location_key)
     if p.target in ('users', 'plugins', 'pipes'):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = f"Cannot sync data to protected table '{p.target}'.",
+            status_code=409,
+            detail=f"Cannot sync data to protected table '{p.target}'.",
         )
 
     if not p.columns and columns is not None:
         p.columns = json.loads(columns)
     if not p.columns and not is_pipe_registered(p, pipes(refresh=True)):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Pipe must be registered with the datetime column specified."
+            status_code=409,
+            detail="Pipe must be registered with index columns specified."
         )
 
     result = list(p.sync(
@@ -412,7 +411,7 @@ def get_pipe_data(
     if params is not None:
         try:
             _params = json.loads(params)
-        except Exception as e:
+        except Exception:
             _params = None
     if not isinstance(_params, dict):
         raise fastapi.HTTPException(
@@ -426,7 +425,7 @@ def get_pipe_data(
     if select_columns is not None:
         try:
             _select_columns = json.loads(select_columns)
-        except Exception as e:
+        except Exception:
             _select_columns = None
     if not isinstance(_select_columns, list):
         raise fastapi.HTTPException(
@@ -440,7 +439,7 @@ def get_pipe_data(
     if omit_columns is not None:
         try:
             _omit_columns = json.loads(omit_columns)
-        except Exception as e:
+        except Exception:
             _omit_columns = None
     if _omit_columns is None:
         raise fastapi.HTTPException(

meerschaum/config/_default.py

@@ -137,7 +137,7 @@ default_pipes_config = {
         },
         'verify': {
             'bound_days': 366,
-            'chunk_minutes': 1440,
+            'chunk_minutes': 43200,
         },
     },
     'attributes': {

meerschaum/config/_paths.py

@@ -103,7 +103,7 @@ if ENVIRONMENT_VENVS_DIR in os.environ:
     if not _VENVS_DIR_PATH.exists():
         try:
             _VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True)
-        except Exception as e:
+        except Exception:
             print(
                 f"Invalid path set for environment variable '{ENVIRONMENT_VENVS_DIR}':\n"
                 + f"{_VENVS_DIR_PATH}"
@@ -148,6 +148,7 @@ paths = {
     'CACHE_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', '.cache'),
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
     'USERS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'users'),
+    'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.16"
+__version__ = "2.7.0"

meerschaum/connectors/api/_pipes.py

@@ -15,7 +15,8 @@ from datetime import datetime
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
-from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
+from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, List, Dict, Tuple
+
 
 def pipe_r_url(
     pipe: mrsm.Pipe
@@ -30,6 +31,7 @@ def pipe_r_url(
         + f"{pipe.connector_keys}/{pipe.metric_key}/{location_key}"
     )
 
+
 def register_pipe(
     self,
     pipe: mrsm.Pipe,
@@ -39,7 +41,6 @@ def register_pipe(
     Returns a tuple of (success_bool, response_dict).
     """
     from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import STATIC_CONFIG
     ### NOTE: if `parameters` is supplied in the Pipe constructor,
     ###       then `pipe.parameters` will exist and not be fetched from the database.
     r_url = pipe_r_url(pipe)
@@ -180,7 +181,7 @@ def sync_pipe(
     from meerschaum.utils.misc import json_serialize_datetime, items_str
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.dataframe import get_numeric_cols, to_json
+    from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
     begin = time.time()
     more_itertools = attempt_import('more_itertools')
     if df is None:
@@ -291,7 +292,7 @@ def sync_pipe(
 
         try:
             j = tuple(j)
-        except Exception as e:
+        except Exception:
             return False, response.text
 
         if debug:
@@ -313,12 +314,12 @@ def sync_pipe(
 
 def delete_pipe(
     self,
-    pipe: Optional[meerschaum.Pipe] = None,
+    pipe: Optional[mrsm.Pipe] = None,
     debug: bool = None,        
 ) -> SuccessTuple:
     """Delete a Pipe and drop its table."""
     if pipe is None:
-        error(f"Pipe cannot be None.")
+        error("Pipe cannot be None.")
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/delete',
@@ -339,7 +340,7 @@ def delete_pipe(
 
 def get_pipe_data(
     self,
-    pipe: meerschaum.Pipe,
+    pipe: mrsm.Pipe,
     select_columns: Optional[List[str]] = None,
     omit_columns: Optional[List[str]] = None,
     begin: Union[str, datetime, int, None] = None,
@@ -351,7 +352,6 @@ def get_pipe_data(
 ) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
     r_url = pipe_r_url(pipe)
-    chunks_list = []
     while True:
         try:
             response = self.get(
@@ -375,12 +375,19 @@ def get_pipe_data(
             return False, j['detail']
         break
 
-    from meerschaum.utils.packages import import_pandas
     from meerschaum.utils.dataframe import parse_df_datetimes, add_missing_cols_to_df
     from meerschaum.utils.dtypes import are_dtypes_equal
-    pd = import_pandas()
     try:
-        df = pd.read_json(StringIO(response.text))
+        df = parse_df_datetimes(
+            j,
+            ignore_cols=[
+                col
+                for col, dtype in pipe.dtypes.items()
+                if not are_dtypes_equal(str(dtype), 'datetime')
+            ],
+            strip_timezone=(pipe.tzinfo is None),
+            debug=debug,
+        )
     except Exception as e:
         warn(f"Failed to parse response for {pipe}:\n{e}")
         return None
@@ -388,16 +395,6 @@ def get_pipe_data(
     if len(df.columns) == 0:
         return add_missing_cols_to_df(df, pipe.dtypes)
 
-    df = parse_df_datetimes(
-        df,
-        ignore_cols = [
-            col
-            for col, dtype in pipe.dtypes.items()
-            if not are_dtypes_equal(str(dtype), 'datetime')
-        ],
-        strip_timezone=(pipe.tzinfo is None),
-        debug=debug,
-    )
     return df
 
 

meerschaum/connectors/sql/_create_engine.py

@@ -107,7 +107,7 @@ flavor_configs = {
         },
     },
     'oracle': {
-        'engine': 'oracle+cx_oracle',
+        'engine': 'oracle+oracledb',
         'create_engine': default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql': {
@@ -164,7 +164,7 @@ install_flavor_drivers = {
     'citus': ['psycopg'],
     'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
-    'oracle': ['cx_Oracle'],
+    'oracle': ['oracledb'],
 }
 require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalchemy_cockroachdb')]}
 
@@ -239,7 +239,7 @@ def create_engine(
             self._sys_config['create_engine'] = {}
         if 'connect_args' not in self._sys_config['create_engine']:
             self._sys_config['create_engine']['connect_args'] = {}
-        self._sys_config['create_engine']['connect_args'].update({"check_same_thread" : False})
+        self._sys_config['create_engine']['connect_args'].update({"check_same_thread": False})
     else:
         engine_str = (
             _engine + "://" + (_username if _username is not None else '') +

meerschaum/connectors/sql/_instance.py

@@ -9,8 +9,7 @@ Define utilities for instance connectors.
 import time
 from datetime import datetime, timezone, timedelta
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, SuccessTuple, Optional, Union, List
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.typing import Dict, SuccessTuple, Union, List
 
 
 _in_memory_temp_tables: Dict[str, bool] = {}
@@ -28,9 +27,9 @@ def _log_temporary_tables_creation(
     from meerschaum.connectors.sql.tables import get_tables
     sqlalchemy = mrsm.attempt_import('sqlalchemy')
     temp_tables_table = get_tables(
-        mrsm_instance = self,
-        create = create,
-        debug = debug,
+        mrsm_instance=self,
+        create=create,
+        debug=debug,
     )['temp_tables']
     if isinstance(tables, str):
         tables = [tables]
@@ -72,7 +71,9 @@ def _drop_temporary_table(
             return True, "Success"
 
     drop_query = f"DROP TABLE {if_exists} " + sql_item_name(
-        table, self.flavor, schema=self.internal_schema
+        table,
+        self.flavor,
+        schema=self.internal_schema
     )
     drop_success = self.exec(drop_query, silent=True, debug=debug) is not None
     drop_msg = "Success" if drop_success else f"Failed to drop temporary table '{table}'."
@@ -83,7 +84,6 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     """
     Drop all tables in the internal schema that are marked as ready to be dropped.
     """
-    from meerschaum.utils.sql import sql_item_name, table_exists
     from meerschaum.utils.misc import items_str
     from meerschaum.connectors.sql.tables import get_tables
     sqlalchemy = mrsm.attempt_import('sqlalchemy')
@@ -141,16 +141,15 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
 
 
 def _drop_old_temporary_tables(
-        self,
-        refresh: bool = True,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    refresh: bool = True,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Drop temporary tables older than the configured interval (24 hours by default).
     """
     from meerschaum.config import get_config
     from meerschaum.connectors.sql.tables import get_tables
-    from meerschaum.utils.misc import items_str
     sqlalchemy = mrsm.attempt_import('sqlalchemy')
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)