meerschaum 2.6.16__py3-none-any.whl → 2.7.0rc1__py3-none-any.whl

meerschaum/utils/dtypes/__init__.py

@@ -15,7 +15,19 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Union, Any
 from meerschaum.utils.warnings import warn
 
-MRSM_PD_DTYPES: Dict[str, str] = {
+MRSM_ALIAS_DTYPES: Dict[str, str] = {
+    'decimal': 'numeric',
+    'number': 'numeric',
+    'jsonl': 'json',
+    'JSON': 'json',
+    'binary': 'bytes',
+    'blob': 'bytes',
+    'varbinary': 'bytes',
+    'bytea': 'bytes',
+    'guid': 'uuid',
+    'UUID': 'uuid',
+}
+MRSM_PD_DTYPES: Dict[Union[str, None], str] = {
     'json': 'object',
     'numeric': 'object',
     'uuid': 'object',
@@ -27,6 +39,8 @@ MRSM_PD_DTYPES: Dict[str, str] = {
     'int32': 'Int32',
     'int64': 'Int64',
     'str': 'string[python]',
+    'bytes': 'object',
+    None: 'object',
 }
 
 
@@ -38,6 +52,10 @@ def to_pandas_dtype(dtype: str) -> str:
     if known_dtype is not None:
         return known_dtype
 
+    alias_dtype = MRSM_ALIAS_DTYPES.get(dtype, None)
+    if alias_dtype is not None:
+        return MRSM_PD_DTYPES[alias_dtype]
+
     ### NOTE: Kind of a hack, but if the first word of the given dtype is in all caps,
     ### treat it as a SQL db type.
     if dtype.split(' ')[0].isupper():
@@ -95,7 +113,7 @@ def are_dtypes_equal(
     try:
         if ldtype == rdtype:
             return True
-    except Exception as e:
+    except Exception:
         warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
         return False
 
@@ -115,6 +133,10 @@ def are_dtypes_equal(
     if ldtype in uuid_dtypes and rdtype in uuid_dtypes:
         return True
 
+    bytes_dtypes = ('bytes', 'object')
+    if ldtype in bytes_dtypes and rdtype in bytes_dtypes:
+        return True
+
     ldtype_clean = ldtype.split('[', maxsplit=1)[0]
     rdtype_clean = rdtype.split('[', maxsplit=1)[0]
 
@@ -185,7 +207,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
             if not value_is_null(value)
             else Decimal('NaN')
         )
-    except Exception as e:
+    except Exception:
         return value
 
 
@@ -201,7 +223,23 @@ def attempt_cast_to_uuid(value: Any) -> Any:
             if not value_is_null(value)
             else None
         )
-    except Exception as e:
+    except Exception:
+        return value
+
+
+def attempt_cast_to_bytes(value: Any) -> Any:
+    """
+    Given a value, attempt to coerce it into a bytestring.
+    """
+    if isinstance(value, uuid.UUID):
+        return value
+    try:
+        return (
+            deserialize_base64(str(value))
+            if not value_is_null(value)
+            else None
+        )
+    except Exception:
         return value
 
 
@@ -251,7 +289,7 @@ def coerce_timezone(
 ) -> Any:
     """
     Given a `datetime`, pandas `Timestamp` or `Series` of `Timestamp`,
-    return a naive datetime in terms of UTC.
+    return a UTC timestamp (strip timezone if `strip_utc` is `True`.
     """
     if dt is None:
         return None
@@ -266,9 +304,7 @@ def coerce_timezone(
     dt_is_series = hasattr(dt, 'dtype') and hasattr(dt, '__module__')
 
     if dt_is_series:
-        is_dask = 'dask' in dt.__module__
         pandas = mrsm.attempt_import('pandas', lazy=False)
-        dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
 
         if (
             pandas.api.types.is_datetime64_any_dtype(dt) and (
@@ -279,14 +315,13 @@ def coerce_timezone(
         ):
             return dt
 
-        dt_series = (
-            pandas.to_datetime(dt, utc=True, format='ISO8601')
-            if dd is None
-            else dd.to_datetime(dt, utc=True, format='ISO8601')
-        )
+        dt_series = to_datetime(dt, coerce_utc=False)
         if strip_utc:
-            if dt_series.dt.tz is not None:
-                dt_series = dt_series.dt.tz_localize(None)
+            try:
+                if dt_series.dt.tz is not None:
+                    dt_series = dt_series.dt.tz_localize(None)
+            except Exception:
+                pass
 
         return dt_series
 
@@ -299,3 +334,57 @@ def coerce_timezone(
     if strip_utc:
         return utc_dt.replace(tzinfo=None)
     return utc_dt
+
+
+def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = True) -> Any:
+    """
+    Wrap `pd.to_datetime()` and add support for out-of-bounds values.
+    """
+    pandas, dateutil_parser = mrsm.attempt_import('pandas', 'dateutil.parser', lazy=False)
+    is_dask = 'dask' in getattr(dt_val, '__module__', '')
+    dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
+    dt_is_series = hasattr(dt_val, 'dtype') and hasattr(dt_val, '__module__')
+    pd = pandas if dd is None else dd
+
+    try:
+        new_dt_val = pd.to_datetime(dt_val, utc=True, format='ISO8601')
+        if as_pydatetime:
+            return new_dt_val.to_pydatetime()
+        return new_dt_val
+    except (pd.errors.OutOfBoundsDatetime, ValueError):
+        pass
+
+    def parse(x: Any) -> Any:
+        try:
+            return dateutil_parser.parse(x)
+        except Exception:
+            return x
+
+    if dt_is_series:
+        new_series = dt_val.apply(parse)
+        if coerce_utc:
+            return coerce_timezone(new_series)
+        return new_series
+
+    new_dt_val = parse(dt_val)
+    if not coerce_utc:
+        return new_dt_val
+    return coerce_timezone(new_dt_val)
+
+
+def serialize_bytes(data: bytes) -> str:
+    """
+    Return the given bytes as a base64-encoded string.
+    """
+    import base64
+    if not isinstance(data, bytes) and value_is_null(data):
+        return data
+    return base64.b64encode(data).decode('utf-8')
+
+
+def deserialize_base64(data: str) -> bytes:
+    """
+    Return the original bytestring from the given base64-encoded string.
+    """
+    import base64
+    return base64.b64decode(data)

meerschaum/utils/dtypes/sql.py

@@ -276,6 +276,19 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'cockroachdb': 'UUID',
         'default': 'TEXT',
     },
+    'bytes': {
+        'timescaledb': 'BYTEA',
+        'postgresql': 'BYTEA',
+        'mariadb': 'BLOB',
+        'mysql': 'BLOB',
+        'mssql': 'VARBINARY(MAX)',
+        'oracle': 'BLOB',
+        'sqlite': 'BLOB',
+        'duckdb': 'BLOB',
+        'citus': 'BYTEA',
+        'cockroachdb': 'BYTEA',
+        'default': 'BLOB',
+    },
 }
 PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
     'int': {
@@ -421,6 +434,19 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'cockroachdb': 'Uuid',
         'default': 'Uuid',
     },
+    'bytes': {
+        'timescaledb': 'LargeBinary',
+        'postgresql': 'LargeBinary',
+        'mariadb': 'LargeBinary',
+        'mysql': 'LargeBinary',
+        'mssql': 'LargeBinary',
+        'oracle': 'LargeBinary',
+        'sqlite': 'LargeBinary',
+        'duckdb': 'LargeBinary',
+        'citus': 'LargeBinary',
+        'cockroachdb': 'LargeBinary',
+        'default': 'LargeBinary',
+    },
 }
 
 AUTO_INCREMENT_COLUMN_FLAVORS: Dict[str, str] = {

meerschaum/utils/misc.py

@@ -177,14 +177,14 @@ def string_to_dict(
         keys = _keys[:-1]
         try:
             val = ast.literal_eval(_keys[-1])
-        except Exception as e:
+        except Exception:
             val = str(_keys[-1])
 
         c = params_dict
         for _k in keys[:-1]:
             try:
                 k = ast.literal_eval(_k)
-            except Exception as e:
+            except Exception:
                 k = str(_k)
             if k not in c:
                 c[k] = {}
@@ -196,12 +196,12 @@ def string_to_dict(
 
 
 def parse_config_substitution(
-        value: str,
-        leading_key: str = 'MRSM',
-        begin_key: str = '{',
-        end_key: str = '}',
-        delimeter: str = ':'
-    ) -> List[Any]:
+    value: str,
+    leading_key: str = 'MRSM',
+    begin_key: str = '{',
+    end_key: str = '}',
+    delimeter: str = ':'
+) -> List[Any]:
     """
     Parse Meerschaum substitution syntax
     E.g. MRSM{value1:value2} => ['value1', 'value2']

meerschaum/utils/packages/_packages.py

@@ -79,7 +79,7 @@ packages: Dict[str, Dict[str, str]] = {
     },
     'drivers-extras': {
         'pyodbc'                     : 'pyodbc>=4.0.30',
-        'cx_Oracle'                  : 'cx_Oracle>=8.3.0',
+        'oracledb'                   : 'oracledb>=2.5.0',
     },
     'cli': {
         'pgcli'                      : 'pgcli>=3.1.0',

meerschaum/utils/schedule.py

@@ -132,7 +132,7 @@ def schedule_function(
 
     try:
         loop.run_until_complete(run_scheduler())
-    except (KeyboardInterrupt, SystemExit) as e:
+    except (KeyboardInterrupt, SystemExit):
         loop.run_until_complete(_stop_scheduler())
 
     return True, "Success"
@@ -159,13 +159,13 @@ def parse_schedule(schedule: str, now: Optional[datetime] = None):
     )
 
     starting_ts = parse_start_time(schedule, now=now)
-    schedule = schedule.split(STARTING_KEYWORD)[0].strip()
+    schedule = schedule.split(STARTING_KEYWORD, maxsplit=1)[0].strip()
     for alias_keyword, true_keyword in SCHEDULE_ALIASES.items():
         schedule = schedule.replace(alias_keyword, true_keyword)
 
     ### TODO Allow for combining `and` + `or` logic.
     if '&' in schedule and '|' in schedule:
-        raise ValueError(f"Cannot accept both 'and' + 'or' logic in the schedule frequency.")
+        raise ValueError("Cannot accept both 'and' + 'or' logic in the schedule frequency.")
 
     join_str = '|' if '|' in schedule else '&'
     join_trigger = (
@@ -300,6 +300,11 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
     try:
         if starting_str == 'now':
             starting_ts = now
+        elif starting_str.startswith('in '):
+            delta_vals = starting_str.replace('in ', '').split(' ', maxsplit=1)
+            delta_unit = delta_vals[-1].rstrip('s') + 's'
+            delta_num = float(delta_vals[0])
+            starting_ts = now + timedelta(**{delta_unit: delta_num})
         elif 'tomorrow' in starting_str or 'today' in starting_str:
             today = round_time(now, timedelta(days=1))
             tomorrow = today + timedelta(days=1)

meerschaum/utils/sql.py

@@ -108,24 +108,30 @@ update_queries = {
             {cols_equal_values}
     """,
     'mssql': """
+        {with_temp_date_bounds}
         MERGE {target_table_name} f
-            USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
+            USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
             ON {and_subquery_f}
             AND {date_bounds_subquery}
         WHEN MATCHED THEN
             UPDATE
             {sets_subquery_none};
     """,
-    'mssql-upsert': """
+    'mssql-upsert': [
+        "{identity_insert_on}",
+        """
+        {with_temp_date_bounds}
         MERGE {target_table_name} f
-            USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
+            USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
             ON {and_subquery_f}
             AND {date_bounds_subquery}
         {when_matched_update_sets_subquery_none}
         WHEN NOT MATCHED THEN
             INSERT ({patch_cols_str})
             VALUES ({patch_cols_prefixed_str});
-    """,
+        """,
+        "{identity_insert_off}",
+    ],
     'oracle': """
         MERGE INTO {target_table_name} f
             USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
@@ -425,20 +431,10 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
         SET seq = {val}
         WHERE name = '{table}'
     """,
-    'oracle': [
-        """
-        DECLARE
-            max_id NUMBER := {val};
-            current_val NUMBER;
-        BEGIN
-            SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
-
-            WHILE current_val < max_id LOOP
-                SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
-            END LOOP;
-        END;
-        """,
-    ],
+    'oracle': (
+        "ALTER TABLE {table_name} MODIFY {column_name} "
+        "GENERATED BY DEFAULT ON NULL AS IDENTITY (START WITH {val_plus_1})"
+    ),
 }
 table_wrappers = {
     'default'    : ('"', '"'),
@@ -1066,7 +1062,7 @@ def get_sqlalchemy_table(
                 connector.metadata,
                 **table_kwargs
             )
-        except sqlalchemy.exc.NoSuchTableError as e:
+        except sqlalchemy.exc.NoSuchTableError:
             warn(f"Table '{truncated_table_name}' does not exist in '{connector}'.")
             return None
     return tables[truncated_table_name]
@@ -1393,6 +1389,7 @@ def get_update_queries(
     datetime_col: Optional[str] = None,
     schema: Optional[str] = None,
     patch_schema: Optional[str] = None,
+    identity_insert: bool = False,
     debug: bool = False,
 ) -> List[str]:
     """
@@ -1430,6 +1427,10 @@ def get_update_queries(
         If provided, use this schema when quoting the patch table.
         Defaults to `schema`.
 
+    identity_insert: bool, default False
+        If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
+        Only applies for MSSQL upserts.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -1564,17 +1565,36 @@ def get_update_queries(
             ) for c_name, c_type in join_cols_types
         ])
 
+    skip_query_val = ""
     target_table_name = sql_item_name(target, flavor, schema)
     patch_table_name = sql_item_name(patch, flavor, patch_schema)
     dt_col_name = sql_item_name(datetime_col, flavor, None) if datetime_col else None
+    date_bounds_table = patch_table_name if flavor != 'mssql' else '[date_bounds]'
+    min_dt_col_name = f"MIN({dt_col_name})" if flavor != 'mssql' else '[Min_dt]'
+    max_dt_col_name = f"MAX({dt_col_name})" if flavor != 'mssql' else '[Max_dt]'
     date_bounds_subquery = (
         f"""
-        f.{dt_col_name} >= (SELECT MIN({dt_col_name}) FROM {patch_table_name})
-        AND f.{dt_col_name} <= (SELECT MAX({dt_col_name}) FROM {patch_table_name})
+        f.{dt_col_name} >= (SELECT {min_dt_col_name} FROM {date_bounds_table})
+        AND f.{dt_col_name} <= (SELECT {max_dt_col_name} FROM {date_bounds_table})
         """
         if datetime_col
         else "1 = 1"
     )
+    with_temp_date_bounds = f"""
+    WITH [date_bounds] AS (
+        SELECT MIN({dt_col_name}) AS {min_dt_col_name}, MAX({dt_col_name}) AS {max_dt_col_name}
+        FROM {patch_table_name}
+    )""" if datetime_col else ""
+    identity_insert_on = (
+        f"SET IDENTITY_INSERT {target_table_name} ON"
+        if identity_insert
+        else skip_query_val
+    )
+    identity_insert_off = (
+        f"SET IDENTITY_INSERT {target_table_name} OFF"
+        if identity_insert
+        else skip_query_val
+    )
 
     ### NOTE: MSSQL upserts must exclude the update portion if only upserting indices.
     when_matched_update_sets_subquery_none = "" if not value_cols else (
@@ -1595,7 +1615,7 @@ def get_update_queries(
     )
     ignore = "IGNORE " if not value_cols else ""
 
-    return [
+    formatted_queries = [
         base_query.format(
             sets_subquery_none=sets_subquery('', 'p.'),
             sets_subquery_none_excluded=sets_subquery('', 'EXCLUDED.'),
@@ -1614,10 +1634,16 @@ def get_update_queries(
             cols_equal_values=cols_equal_values,
             on_duplicate_key_update=on_duplicate_key_update,
             ignore=ignore,
+            with_temp_date_bounds=with_temp_date_bounds,
+            identity_insert_on=identity_insert_on,
+            identity_insert_off=identity_insert_off,
         )
         for base_query in base_queries
     ]
 
+    ### NOTE: Allow for skipping some queries.
+    return [query for query in formatted_queries if query]
+
 
 def get_null_replacement(typ: str, flavor: str) -> str:
     """
@@ -1867,7 +1893,17 @@ def _get_create_table_query_from_dtypes(
 
     table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
     primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
+    primary_key_constraint_name = (
+        sql_item_name(f'PK_{new_table}', flavor, None)
+        if primary_key
+        else None
+    )
     datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
+    primary_key_clustered = (
+        "CLUSTERED"
+        if not datetime_column or datetime_column == primary_key
+        else "NONCLUSTERED"
+    )
     query = f"CREATE TABLE {table_name} ("
     if primary_key:
         col_db_type = cols_types[0][1]
@@ -1887,6 +1923,8 @@ def _get_create_table_query_from_dtypes(
             query += f"\n    {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
         elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
             query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
+        elif flavor == 'mssql':
+            query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
         else:
             query += f"\n    {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
 
@@ -1902,6 +1940,10 @@ def _get_create_table_query_from_dtypes(
         and datetime_column != primary_key
     ):
         query += f"\n    PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
+
+    if flavor == 'mssql' and primary_key:
+        query += f"\n    CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name}),"
+
     query = query[:-1]
     query += "\n)"
 
@@ -1927,7 +1969,7 @@ def _get_create_table_query_from_cte(
     create_cte_name = sql_item_name(create_cte, flavor, None)
     new_table_name = sql_item_name(new_table, flavor, schema)
     primary_key_constraint_name = (
-        sql_item_name(f'pk_{new_table}', flavor, None)
+        sql_item_name(f'PK_{new_table}', flavor, None)
         if primary_key
         else None
     )
@@ -1936,6 +1978,7 @@ def _get_create_table_query_from_cte(
         if primary_key
         else None
     )
+    primary_key_clustered = "CLUSTERED" if not datetime_column else "NONCLUSTERED"
     datetime_column_name = (
         sql_item_name(datetime_column, flavor)
         if datetime_column
@@ -1943,7 +1986,7 @@ def _get_create_table_query_from_cte(
     )
     if flavor in ('mssql',):
         query = query.lstrip()
-        if 'with ' in query.lower():
+        if query.lower().startswith('with '):
             final_select_ix = query.lower().rfind('select')
             create_table_query = (
                 query[:final_select_ix].rstrip() + ',\n'
@@ -1961,7 +2004,7 @@ def _get_create_table_query_from_cte(
 
         alter_type_query = f"""
             ALTER TABLE {new_table_name}
-            ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})
+            ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name})
         """
     elif flavor in (None,):
         create_table_query = f"""
@@ -2225,29 +2268,8 @@ def get_reset_autoincrement_queries(
     schema = schema or connector.schema
     max_id_name = sql_item_name('max_id', connector.flavor)
     table_name = sql_item_name(table, connector.flavor, schema)
-    table_trunc = truncate_item_name(table, connector.flavor)
     table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
     column_name = sql_item_name(column, connector.flavor)
-    if connector.flavor == 'oracle':
-        potential_table_names = set([
-            f"'{table_trunc.upper()}'",
-            f"'{table_trunc}'",
-            f"'{table_name}'",
-            f"'{table_name.upper()}'",
-        ])
-        df = connector.read(
-            """
-            SELECT SEQUENCE_NAME
-            FROM ALL_TAB_IDENTITY_COLS
-            WHERE TABLE_NAME IN ("""
-            + ", ".join([name for name in potential_table_names])
-            + """)
-            """,
-            debug=debug
-        )
-        if len(df) > 0:
-            table_seq_name = df['sequence_name'][0]
-
     max_id = connector.value(
         f"""
         SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
@@ -2272,7 +2294,8 @@ def get_reset_autoincrement_queries(
             table=table,
             table_name=table_name,
             table_seq_name=table_seq_name,
-            val=(max_id),
+            val=max_id,
+            val_plus_1=(max_id + 1),
         )
         for query in reset_queries
     ]

meerschaum/utils/venv/_Venv.py

@@ -34,10 +34,10 @@ class Venv:
     """
 
     def __init__(
-            self,
-            venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
-            debug: bool = False,
-        ) -> None:
+        self,
+        venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
+        debug: bool = False,
+    ) -> None:
         from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
         ### For some weird threading issue,
         ### we can't use `isinstance` here.