meerschaum 2.6.14__py3-none-any.whl → 2.6.16__py3-none-any.whl

meerschaum/actions/bootstrap.py

@@ -54,10 +54,9 @@ def _bootstrap_pipes(
     """
     from meerschaum import get_pipes
     from meerschaum.config import get_config
-    from meerschaum.utils.warnings import info, warn, error
-    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.prompt import yes_no, prompt, choose
-    from meerschaum.connectors.parse import is_valid_connector_keys, parse_instance_keys
+    from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.misc import get_connector_labels
     from meerschaum.utils.formatting._shell import clear_screen
 
@@ -102,7 +101,7 @@ def _bootstrap_pipes(
         ### Get the connector.
         new_label = 'New'
         info(
-            f"To create a pipe without explicitly using a connector, "
+            "To create a pipe without explicitly using a connector, "
             + "use the `register pipes` command.\n"
         )
         try:
@@ -133,7 +132,7 @@ def _bootstrap_pipes(
                     break
                 elif isinstance(tup[0], bool) and not tup[0]:
                     return abort_tuple
-                warn(f"Please register a new connector or press CTRL+C to cancel.", stack=False)
+                warn("Please register a new connector or press CTRL+C to cancel.", stack=False)
         connector_keys = [ck]
 
         ### Get the metric.
@@ -142,9 +141,9 @@ def _bootstrap_pipes(
                 clear_screen(debug=debug)
             try:
                 mk = prompt(
-                    f"What kind of data is this?\n\n" +
-                    f"    The metric is the label for the contents of the pipe.\n" +
-                    f"    For example, 'weather' might be a metric for weather station data.\n\n" +
+                    "What kind of data is this?\n\n" +
+                    "    The metric is the label for the contents of the pipe.\n" +
+                    "    For example, 'weather' might be a metric for weather station data.\n\n" +
                     f" {get_config('formatting', 'emoji', 'metric')} Metric:"
                 )
             except KeyboardInterrupt:
@@ -224,6 +223,7 @@ def _bootstrap_pipes(
 
     return (successes > 0), msg
 
+
 def _bootstrap_connectors(
     action: Optional[List[str]] = None,
     connector_keys: Optional[List[str]] = None,
@@ -237,7 +237,6 @@ def _bootstrap_connectors(
     """
     Prompt the user for the details necessary to create a Connector.
     """
-    from meerschaum.connectors.parse import is_valid_connector_keys
     from meerschaum.connectors import (
         connectors,
         get_connector,

meerschaum/actions/copy.py

@@ -38,7 +38,6 @@ def _complete_copy(
     """
     Override the default Meerschaum `complete_` function.
     """
-    from meerschaum.actions.edit import _complete_edit_config
     if action is None:
         action = []
 
@@ -71,7 +70,7 @@ def _copy_pipes(
     """
     from meerschaum import get_pipes, Pipe
     from meerschaum.utils.prompt import prompt, yes_no
-    from meerschaum.utils.warnings import warn, info
+    from meerschaum.utils.warnings import warn
     from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.formatting._shell import clear_screen
     pipes = get_pipes(as_list=True, **kw)
@@ -110,7 +109,9 @@ def _copy_pipes(
                     f"Do you want to copy data from {p} into {_new_pipe}?\n\n"
                     + "If you specified `--begin`, `--end` or `--params`, data will be filtered."
                 ),
-                    noask=noask, yes=yes
+                    noask=noask,
+                    yes=yes,
+                    default='n',
                 )
         ):
             _new_pipe.sync(

meerschaum/config/_default.py

@@ -131,7 +131,6 @@ default_pipes_config = {
     'parameters': {
         'columns': {
             'datetime': None,
-            'id': None,
         },
         'fetch': {
             'backtrack_minutes': 1440,

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.14"
+__version__ = "2.6.16"

meerschaum/connectors/sql/_fetch.py

@@ -284,15 +284,15 @@ def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
     - query
     - sql
     """
-    if pipe.parameters.get('fetch', {}).get('definition', None):
+    if 'fetch' in pipe.parameters and 'definition' in pipe.parameters['fetch']:
         if pipe.parameters.get('fetch', None) is None:
             pipe.parameters['fetch'] = {}
         dict_to_set = pipe.parameters['fetch']
         key_to_set = 'definition'
-    elif pipe.parameters.get('definition', None):
+    elif 'definition' in pipe.parameters:
         dict_to_set = pipe.parameters
         key_to_set = 'definition'
-    elif pipe.parameters.get('query', None):
+    elif 'query' in pipe.parameters:
         dict_to_set = pipe.parameters
         key_to_set = 'query'
     else:

meerschaum/connectors/sql/_pipes.py

@@ -1770,8 +1770,8 @@ def sync_pipe_inplace(
     self,
     pipe: 'mrsm.Pipe',
     params: Optional[Dict[str, Any]] = None,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
     chunksize: Optional[int] = -1,
     check_existing: bool = True,
     debug: bool = False,
@@ -1790,11 +1790,11 @@ def sync_pipe_inplace(
         Optional params dictionary to build the `WHERE` clause.
         See `meerschaum.utils.sql.build_where`.
 
-    begin: Optional[datetime], default None
+    begin: Union[datetime, int, None], default None
         Optionally specify the earliest datetime to search for data.
         Defaults to `None`.
 
-    end: Optional[datetime], default None
+    end: Union[datetime, int, None], default None
         Optionally specify the latest datetime to search for data.
         Defaults to `None`.
 
@@ -1834,7 +1834,7 @@ def sync_pipe_inplace(
         session_execute,
         update_queries,
     )
-    from meerschaum.utils.dtypes import coerce_timezone, are_dtypes_equal
+    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
         get_pd_type_from_db_type,
     )
@@ -1842,7 +1842,7 @@ def sync_pipe_inplace(
 
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
-        temp_prefix = '##' if self.flavor != 'oracle' else ''
+        temp_prefix = '##' if self.flavor != 'oracle' else '_'
         return temp_prefix + transact_id + '_' + label + '_' + pipe.target
 
     internal_schema = self.internal_schema
@@ -1875,7 +1875,6 @@ def sync_pipe_inplace(
     autoincrement = pipe.parameters.get('autoincrement', False)
     dt_col = pipe.columns.get('datetime', None)
     dt_col_name = sql_item_name(dt_col, self.flavor, None) if dt_col else None
-    dt_typ = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]') if dt_col else None
 
     def clean_up_temp_tables(ready_to_drop: bool = False):
         log_success, log_msg = self._log_temporary_tables_creation(
@@ -2008,14 +2007,14 @@ def sync_pipe_inplace(
         ],
         with_results=True,
         debug=debug,
-    ) if not upsert else ((True, "Success"), None)
+    ) if dt_col and not upsert else ((True, "Success"), None)
     if not new_dt_bounds_success:
         return (
             new_dt_bounds_success,
             f"Could not determine in-place datetime bounds:\n{new_dt_bounds_msg}"
         )
 
-    if not upsert:
+    if dt_col and not upsert:
         begin, end = new_dt_bounds_results[0].fetchone()
 
     backtrack_def = self.get_pipe_data_query(
@@ -2352,18 +2351,9 @@ def get_sync_time(
     table = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
 
     dt_col = pipe.columns.get('datetime', None)
-    dt_type = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]')
-    if not dt_col:
-        _dt = pipe.guess_datetime()
-        dt = sql_item_name(_dt, self.flavor, None) if _dt else None
-        is_guess = True
-    else:
-        _dt = dt_col
-        dt = sql_item_name(_dt, self.flavor, None)
-        is_guess = False
-
-    if _dt is None:
+    if dt_col is None:
         return None
+    dt_col_name = sql_item_name(dt_col, self.flavor, None)
 
     ASC_or_DESC = "DESC" if newest else "ASC"
     existing_cols = pipe.get_columns_types(debug=debug)
@@ -2373,16 +2363,17 @@ def get_sync_time(
 
     ### If no bounds are provided for the datetime column,
     ### add IS NOT NULL to the WHERE clause.
-    if _dt not in valid_params:
-        valid_params[_dt] = '_None'
+    if dt_col not in valid_params:
+        valid_params[dt_col] = '_None'
     where = "" if not valid_params else build_where(valid_params, self)
-    q = f"SELECT {dt}\nFROM {table}{where}\nORDER BY {dt} {ASC_or_DESC}\nLIMIT 1"
+    q = f"SELECT {dt_col_name}\nFROM {table}{where}\nORDER BY {dt_col_name} {ASC_or_DESC}\nLIMIT 1"
     if self.flavor == 'mssql':
-        q = f"SELECT TOP 1 {dt}\nFROM {table}{where}\nORDER BY {dt} {ASC_or_DESC}"
+        q = f"SELECT TOP 1 {dt_col_name}\nFROM {table}{where}\nORDER BY {dt_col_name} {ASC_or_DESC}"
     elif self.flavor == 'oracle':
         q = (
             "SELECT * FROM (\n"
-            + f"    SELECT {dt}\nFROM {table}{where}\n    ORDER BY {dt} {ASC_or_DESC}\n"
+            + f"    SELECT {dt_col_name}\nFROM {table}{where}\n    "
+            + f"ORDER BY {dt_col_name} {ASC_or_DESC}\n"
             + ") WHERE ROWNUM = 1"
         )
 

meerschaum/core/Pipe/_attributes.py

@@ -612,24 +612,34 @@ def target(self) -> str:
       - `target_table_name`
     """
     if 'target' not in self.parameters:
-        target = self._target_legacy()
+        default_target = self._target_legacy()
+        default_targets = {default_target}
         potential_keys = ('target_name', 'target_table', 'target_table_name')
+        _target = None
         for k in potential_keys:
             if k in self.parameters:
-                target = self.parameters[k]
+                _target = self.parameters[k]
                 break
 
+        _target = _target or default_target
+
         if self.instance_connector.type == 'sql':
             from meerschaum.utils.sql import truncate_item_name
-            truncated_target = truncate_item_name(target, self.instance_connector.flavor)
-            if truncated_target != target:
-                warn(
-                    f"The target '{target}' is too long for '{self.instance_connector.flavor}', "
-                    + f"will use {truncated_target} instead."
-                )
-                target = truncated_target
-
-        self.target = target
+            truncated_target = truncate_item_name(_target, self.instance_connector.flavor)
+            default_targets.add(truncated_target)
+            warned_target = self.__dict__.get('_warned_target', False)
+            if truncated_target != _target and not warned_target:
+                if not warned_target:
+                    warn(
+                        f"The target '{_target}' is too long for '{self.instance_connector.flavor}', "
+                        + f"will use {truncated_target} instead."
+                    )
+                    self.__dict__['_warned_target'] = True
+                _target = truncated_target
+
+        if _target in default_targets:
+            return _target
+        self.target = _target
     return self.parameters['target']
 
 

meerschaum/core/Pipe/_bootstrap.py

@@ -7,17 +7,18 @@ Attempt to create a pipe's requirements in one method.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, Dict, Any
+from meerschaum.utils.typing import SuccessTuple, Dict, Any
+
 
 def bootstrap(
-        self,
-        debug: bool = False,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        shell: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    shell: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Prompt the user to create a pipe's requirements all from one method.
     This method shouldn't be used in any automated scripts because it interactively
@@ -46,7 +47,7 @@ def bootstrap(
 
     """
 
-    from meerschaum.utils.warnings import warn, info, error
+    from meerschaum.utils.warnings import info
     from meerschaum.utils.prompt import prompt, yes_no
     from meerschaum.utils.formatting import pprint
     from meerschaum.config import get_config
@@ -74,8 +75,8 @@ def bootstrap(
             f"\n    Press [Enter] to register {self} with the above configuration:",
             icon = False
         )
-    except KeyboardInterrupt as e:
-        return False, f"Aborting bootstrapping {self}."
+    except KeyboardInterrupt:
+        return False, f"Aborted bootstrapping {self}."
 
     with Venv(get_connector_plugin(self.instance_connector)):
         register_tuple = self.instance_connector.register_pipe(self, debug=debug)
@@ -119,7 +120,7 @@ def bootstrap(
 
     print_tuple((True, f"Finished bootstrapping {self}!"))
     info(
-        f"You can edit this pipe later with `edit pipes` "
+        "You can edit this pipe later with `edit pipes` "
         + "or set the definition with `edit pipes definition`.\n"
         + "    To sync data into your pipe, run `sync pipes`."
     )
@@ -128,7 +129,7 @@ def bootstrap(
 
 
 def _get_parameters(pipe, debug: bool = False) -> Dict[str, Any]:
-    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.prompt import prompt
     from meerschaum.utils.warnings import warn, info
     from meerschaum.config import get_config
     from meerschaum.config._patch import apply_patch_to_config
@@ -139,36 +140,22 @@ def _get_parameters(pipe, debug: bool = False) -> Dict[str, Any]:
     _types_defaults = {
         'sql': {
             'fetch': {
+                'backtrack_minutes': get_config(
+                    'pipes', 'parameters', 'fetch', 'backtrack_minutes'
+                ),
                 'definition': None,
             },
-            'parents': [
-                {
-                    'connector': None,
-                    'metric': None,
-                    'location': None,
-                    'instance': None,
-                },
-            ],
+            'verify': {
+                'chunk_minutes': get_config('pipes', 'parameters', 'verify', 'chunk_minutes'),
+            },
         },
         'api': {
-            'fetch': {
-                'connector': None,
-                'metric': None,
-                'location': None,
-            },
-            'parents': [
-                {
-                    'connector': None,
-                    'metric': None,
-                    'location': None,
-                    'instance': None,
-                },
-            ],
+            'fetch': {},
         },
     }
     try:
         conn_type = pipe.connector.type
-    except Exception as e:
+    except Exception:
         conn_type = None
     _parameters = _types_defaults.get(conn_type, {})
 
@@ -210,7 +197,7 @@ def _get_parameters(pipe, debug: bool = False) -> Dict[str, Any]:
         _parameters['tags'] = [
             t.strip() for t in prompt(f"Tags for {pipe} (empty to omit):").split(',') if t
         ]
-    except Exception as e:
+    except Exception:
         _parameters['tags'] = []
 
     return _parameters

meerschaum/core/Pipe/_sync.py

@@ -493,6 +493,9 @@ def get_sync_time(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import round_time
 
+    if not self.columns.get('datetime', None):
+        return None
+
     with Venv(get_connector_plugin(self.instance_connector)):
         sync_time = self.instance_connector.get_sync_time(
             self,

meerschaum/utils/prompt.py

@@ -127,15 +127,15 @@ def prompt(
 
 
 def yes_no(
-        question: str = '',
-        options: Tuple[str, str] = ('y', 'n'),
-        default: str = 'y',
-        wrappers: Tuple[str, str] = ('[', ']'),
-        icon: bool = True,
-        yes: bool = False,
-        noask: bool = False,
-        **kw : Any
-    ) -> bool:
+    question: str = '',
+    options: Tuple[str, str] = ('y', 'n'),
+    default: str = 'y',
+    wrappers: Tuple[str, str] = ('[', ']'),
+    icon: bool = True,
+    yes: bool = False,
+    noask: bool = False,
+    **kw : Any
+) -> bool:
     """
     Print a question and prompt the user with a yes / no input.
     Returns `True` for `'yes'`, False for `'no'`.

meerschaum/utils/sql.py

@@ -2229,11 +2229,22 @@ def get_reset_autoincrement_queries(
     table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
     column_name = sql_item_name(column, connector.flavor)
     if connector.flavor == 'oracle':
-        df = connector.read(f"""
+        potential_table_names = set([
+            f"'{table_trunc.upper()}'",
+            f"'{table_trunc}'",
+            f"'{table_name}'",
+            f"'{table_name.upper()}'",
+        ])
+        df = connector.read(
+            """
             SELECT SEQUENCE_NAME
             FROM ALL_TAB_IDENTITY_COLS
-            WHERE TABLE_NAME IN '{table_trunc.upper()}'
-        """, debug=debug)
+            WHERE TABLE_NAME IN ("""
+            + ", ".join([name for name in potential_table_names])
+            + """)
+            """,
+            debug=debug
+        )
         if len(df) > 0:
             table_seq_name = df['sequence_name'][0]