meerschaum 2.4.5__py3-none-any.whl → 2.4.7__py3-none-any.whl

meerschaum/_internal/docs/index.py

@@ -545,6 +545,7 @@ def init_dash(dash_app):
       <li><code>meerschaum.utils.dataframe.get_unhashable_cols()</code></li>
       <li><code>meerschaum.utils.dataframe.parse_df_datetimes()</code></li>
       <li><code>meerschaum.utils.dataframe.query_df()</code></li>
+      <li><code>meerschaum.utils.dataframe.to_json()</code></li>
     </ul>
   </details>
   </ul>

meerschaum/actions/show.py

@@ -309,6 +309,7 @@ def _show_data(
     from meerschaum.utils.packages import attempt_import, import_pandas
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.formatting import pprint
+    from meerschaum.utils.dataframe import to_json
     pd = import_pandas()
 
     if action is None:
@@ -372,7 +373,7 @@ def _show_data(
             info(info_msg)
         else:
             print(json.dumps(p.__getstate__()))
-            df = df.fillna(pd.NA).to_json(orient='columns')
+            df = to_json(df, orient='columns')
 
         pprint(df, nopretty=nopretty)
 

meerschaum/actions/sql.py

@@ -15,12 +15,12 @@ exec_methods = {
 }
 
 def sql(
-        action: Optional[List[str]] = None,
-        gui: bool = False,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ):
+    action: Optional[List[str]] = None,
+    gui: bool = False,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+):
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
     
     Usage:
@@ -59,6 +59,7 @@ def sql(
         - `sql local exec "INSERT INTO table (id) VALUES (1)"
             Execute the above query on `sql:local`.
     """
+    from meerschaum.utils.dataframe impor to_json
 
     if action is None:
         action = []
@@ -135,11 +136,10 @@ def sql(
             pprint(result)
         else:
             print(
-                result.fillna(pd.NA).to_json(
-                    date_format = 'iso',
-                    orient = 'split',
-                    index = False,
-                    date_unit = 'us'
+                to_json(
+                    result,
+                    orient='split',
+                    index=False,
                 )
             )
 

meerschaum/api/dash/pipes.py

@@ -18,6 +18,7 @@ from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
 from meerschaum.utils.sql import get_pd_type
 from meerschaum.utils.yaml import yaml
+from meerschaum.utils.dataframe import to_json
 from meerschaum.connectors.sql._fetch import get_pipe_query
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.api.dash import debug, _get_pipes
@@ -563,12 +564,13 @@ def accordion_items_from_pipe(
     if 'sync-data' in active_items:
         backtrack_df = pipe.get_backtrack_data(debug=debug, limit=1)
         try:
-            json_text = backtrack_df.fillna(pd.NA).to_json(
+            json_text = to_json(
+                backtrack_df,
                 orient='records',
                 date_format='iso',
                 force_ascii=False,
                 indent=4,
-                date_unit='ns',
+                date_unit='us',
             ) if backtrack_df is not None else '[]'
         except Exception as e:
             warn(e)

meerschaum/api/routes/_pipes.py

@@ -27,7 +27,7 @@ from decimal import Decimal
 from meerschaum import Pipe
 from meerschaum.api.models import MetaPipe
 from meerschaum.utils.packages import attempt_import, import_pandas
-from meerschaum.utils.dataframe import get_numeric_cols
+from meerschaum.utils.dataframe import get_numeric_cols, to_json
 from meerschaum.utils.misc import (
     is_pipe_registered, round_time, is_int, parse_df_datetimes,
     replace_pipes_in_dict,
@@ -473,15 +473,10 @@ def get_pipe_data(
     for col in numeric_cols:
         df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
 
-    json_content = df.to_json(
-        date_format='iso',
-        orient='records',
-        date_unit='us',
-    )
-
+    json_content = to_json(df)
     return fastapi.Response(
         json_content,
-        media_type = 'application/json',
+        media_type='application/json',
     )
 
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.5"
+__version__ = "2.4.7"

meerschaum/connectors/api/_pipes.py

@@ -171,8 +171,8 @@ def sync_pipe(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.misc import json_serialize_datetime, items_str
     from meerschaum.config import get_config
-    from meerschaum.utils.packages import attempt_import, import_pandas
-    from meerschaum.utils.dataframe import get_numeric_cols
+    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.dataframe import get_numeric_cols, to_json
     begin = time.time()
     more_itertools = attempt_import('more_itertools')
     if df is None:
@@ -183,8 +183,7 @@ def sync_pipe(
         ### allow syncing dict or JSON without needing to import pandas (for IOT devices)
         if isinstance(c, (dict, list)):
             return json.dumps(c, default=json_serialize_datetime)
-        pd = import_pandas()
-        return c.fillna(pd.NA).to_json(date_format='iso', date_unit='ns')
+        return to_json(c, orient='columns')
 
     df = json.loads(df) if isinstance(df, str) else df
 
@@ -202,6 +201,7 @@ def sync_pipe(
             if not is_dask
             else [partition.compute() for partition in df.partitions]
         )
+
         numeric_cols = get_numeric_cols(df)
         if numeric_cols:
             for col in numeric_cols:

meerschaum/connectors/sql/_SQLConnector.py

@@ -13,6 +13,7 @@ from meerschaum.utils.typing import Optional, Any, Union
 from meerschaum.connectors import Connector
 from meerschaum.utils.warnings import error
 
+
 class SQLConnector(Connector):
     """
     Connect to SQL databases via `sqlalchemy`.
@@ -26,7 +27,16 @@ class SQLConnector(Connector):
     IS_INSTANCE: bool = True
 
     from ._create_engine import flavor_configs, create_engine
-    from ._sql import read, value, exec, execute, to_sql, exec_queries
+    from ._sql import (
+        read,
+        value,
+        exec,
+        execute,
+        to_sql,
+        exec_queries,
+        get_connection,
+        _cleanup_connections,
+    )
     from meerschaum.utils.sql import test_connection
     from ._fetch import fetch, get_pipe_metadef
     from ._cli import cli, _cli_exit
@@ -85,7 +95,7 @@ class SQLConnector(Connector):
         _drop_temporary_tables,
         _drop_old_temporary_tables,
     )
-    
+
     def __init__(
         self,
         label: Optional[str] = None,

meerschaum/connectors/sql/_create_engine.py

@@ -71,7 +71,13 @@ flavor_configs = {
         'requirements': default_requirements,
         'defaults': {
             'port': 1433,
-            'options': "driver=ODBC Driver 18 for SQL Server&UseFMTONLY=Yes&TrustServerCertificate=yes&Encrypt=no",
+            'options': (
+                "driver=ODBC Driver 18 for SQL Server"
+                "&UseFMTONLY=Yes"
+                "&TrustServerCertificate=yes"
+                "&Encrypt=no"
+                "&MARS_Connection=yes"
+            ),
         },
     },
     'mysql': {
@@ -167,12 +173,13 @@ flavor_dialects = {
     'duckdb': ('duckdb', 'duckdb_engine', 'Dialect'),
 }
 
+
 def create_engine(
-        self,
-        include_uri: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> 'sqlalchemy.engine.Engine':
+    self,
+    include_uri: bool = False,
+    debug: bool = False,
+    **kw
+) -> 'sqlalchemy.engine.Engine':
     """Create a sqlalchemy engine by building the engine string."""
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import error, warn

meerschaum/connectors/sql/_pipes.py

@@ -11,9 +11,10 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import (
     Union, Any, SuccessTuple, Tuple, Dict, Optional, List
 )
-from meerschaum.utils.warnings import warn, error, info
+from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.debug import dprint
 
+
 def register_pipe(
     self,
     pipe: mrsm.Pipe,
@@ -73,12 +74,12 @@ def register_pipe(
 
 
 def edit_pipe(
-        self,
-        pipe : mrsm.Pipe = None,
-        patch: bool = False,
-        debug: bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    self,
+    pipe : mrsm.Pipe = None,
+    patch: bool = False,
+    debug: bool = False,
+    **kw : Any
+) -> SuccessTuple:
     """
     Persist a Pipe's parameters to its database.
 
@@ -307,11 +308,11 @@ def fetch_pipes_keys(
 
 
 def create_indices(
-        self,
-        pipe: mrsm.Pipe,
-        indices: Optional[List[str]] = None,
-        debug: bool = False
-    ) -> bool:
+    self,
+    pipe: mrsm.Pipe,
+    indices: Optional[List[str]] = None,
+    debug: bool = False
+) -> bool:
     """
     Create a pipe's indices.
     """
@@ -339,11 +340,11 @@ def create_indices(
 
 
 def drop_indices(
-        self,
-        pipe: mrsm.Pipe,
-        indices: Optional[List[str]] = None,
-        debug: bool = False
-    ) -> bool:
+    self,
+    pipe: mrsm.Pipe,
+    indices: Optional[List[str]] = None,
+    debug: bool = False
+) -> bool:
     """
     Drop a pipe's indices.
     """
@@ -369,10 +370,10 @@ def drop_indices(
 
 
 def get_create_index_queries(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, List[str]]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, List[str]]:
     """
     Return a dictionary mapping columns to a `CREATE INDEX` or equivalent query.
 
@@ -541,10 +542,10 @@ def get_create_index_queries(
 
 
 def get_drop_index_queries(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, List[str]]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, List[str]]:
     """
     Return a dictionary mapping columns to a `DROP INDEX` or equivalent query.
 
@@ -607,10 +608,10 @@ def get_drop_index_queries(
 
 
 def delete_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Delete a Pipe's registration.
     """
@@ -701,7 +702,7 @@ def get_pipe_data(
     from meerschaum.utils.sql import sql_item_name
     from meerschaum.utils.misc import parse_df_datetimes, to_pandas_dtype
     from meerschaum.utils.packages import import_pandas
-    from meerschaum.utils.dtypes import attempt_cast_to_numeric
+    from meerschaum.utils.dtypes import attempt_cast_to_numeric, attempt_cast_to_uuid
     pd = import_pandas()
     is_dask = 'dask' in pd.__name__
 
@@ -768,6 +769,12 @@ def get_pipe_data(
         for col, typ in pipe.dtypes.items()
         if typ == 'numeric' and col in dtypes
     ]
+    uuid_columns = [
+        col
+        for col, typ in pipe.dtypes.items()
+        if typ == 'uuid' and col in dtypes
+    ]
+
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
 
     df = self.read(
@@ -781,6 +788,11 @@ def get_pipe_data(
             continue
         df[col] = df[col].apply(attempt_cast_to_numeric)
 
+    for col in uuid_columns:
+        if col not in df.columns:
+            continue
+        df[col] = df[col].apply(attempt_cast_to_uuid)
+
     if self.flavor == 'sqlite':
         ignore_dt_cols = [
             col
@@ -1036,10 +1048,10 @@ def get_pipe_data_query(
 
 
 def get_pipe_id(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Any:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Any:
     """
     Get a Pipe's ID from the pipes table.
     """
@@ -1066,10 +1078,10 @@ def get_pipe_id(
 
 
 def get_pipe_attributes(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, Any]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """
     Get a Pipe's attributes dictionary.
     """
@@ -1236,7 +1248,7 @@ def sync_pipe(
         }
         pipe_dtypes.update(new_bool_cols)
         pipe.dtypes = pipe_dtypes
-        if not pipe.temporary:
+        if new_bool_cols and not pipe.temporary:
             infer_bool_success, infer_bool_msg = pipe.edit(debug=debug)
             if not infer_bool_success:
                 return infer_bool_success, infer_bool_msg
@@ -1468,7 +1480,7 @@ def sync_pipe_inplace(
 
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
-        return '-' + transact_id + '_' + label + '_' + pipe.target
+        return '##' + transact_id + '_' + label + '_' + pipe.target
 
     internal_schema = self.internal_schema
     temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
@@ -1486,11 +1498,11 @@ def sync_pipe_inplace(
     }
     metadef = self.get_pipe_metadef(
         pipe,
-        params = params,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
+        params=params,
+        begin=begin,
+        end=end,
+        check_existing=check_existing,
+        debug=debug,
     )
     pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
     upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
@@ -1502,15 +1514,15 @@ def sync_pipe_inplace(
                 table
                 for table in temp_tables.values()
             ] if not upsert else [temp_tables['update']],
-            ready_to_drop = ready_to_drop,
-            create = (not pipe.temporary),
-            debug = debug,
+            ready_to_drop=ready_to_drop,
+            create=(not pipe.temporary),
+            debug=debug,
         )
         if not log_success:
             warn(log_msg)
         drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(
-            refresh = False,
-            debug = debug,
+            refresh=False,
+            debug=debug,
         )
         if not drop_stale_success:
             warn(drop_stale_msg)
@@ -1522,7 +1534,7 @@ def sync_pipe_inplace(
             metadef,
             pipe.target,
             self.flavor,
-            schema = self.get_pipe_schema(pipe),
+            schema=self.get_pipe_schema(pipe),
         )
         result = self.exec(create_pipe_query, debug=debug)
         if result is None:
@@ -1543,13 +1555,13 @@ def sync_pipe_inplace(
         metadef,
         temp_tables[('new') if not upsert else 'update'],
         self.flavor,
-        schema = internal_schema,
+        schema=internal_schema,
     )
     (create_new_success, create_new_msg), create_new_results = session_execute(
         session,
         create_new_query,
-        with_results = True,
-        debug = debug,
+        with_results=True,
+        debug=debug,
     )
     if not create_new_success:
         _ = clean_up_temp_tables()
@@ -1558,11 +1570,11 @@ def sync_pipe_inplace(
 
     new_cols_types = get_table_cols_types(
         temp_tables[('new' if not upsert else 'update')],
-        connectable = connectable,
-        flavor = self.flavor,
-        schema = internal_schema,
-        database = database,
-        debug = debug,
+        connectable=connectable,
+        flavor=self.flavor,
+        schema=internal_schema,
+        database=database,
+        debug=debug,
     )
     if not new_cols_types:
         return False, f"Failed to get new columns for {pipe}."
@@ -2389,6 +2401,7 @@ def get_add_columns_queries(
     self,
     pipe: mrsm.Pipe,
     df: Union[pd.DataFrame, Dict[str, str]],
+    _is_db_types: bool = False,
     debug: bool = False,
 ) -> List[str]:
     """
@@ -2403,6 +2416,9 @@ def get_add_columns_queries(
         The pandas DataFrame which contains new columns.
         If a dictionary is provided, assume it maps columns to Pandas data types.
 
+    _is_db_types: bool, default False
+        If `True`, assume `df` is a dictionary mapping columns to SQL native dtypes.
+
     Returns
     -------
     A list of the `ALTER TABLE` SQL query or queries to be executed on the provided connector.
@@ -2495,11 +2511,11 @@ def get_add_columns_queries(
 
 
 def get_alter_columns_queries(
-        self,
-        pipe: mrsm.Pipe,
-        df: Union[pd.DataFrame, Dict[str, str]],
-        debug: bool = False,
-    ) -> List[str]:
+    self,
+    pipe: mrsm.Pipe,
+    df: Union[pd.DataFrame, Dict[str, str]],
+    debug: bool = False,
+) -> List[str]:
     """
     If we encounter a column of a different type, set the entire column to text.
     If the altered columns are numeric, alter to numeric instead.
@@ -2556,6 +2572,7 @@ def get_alter_columns_queries(
         'int': 'bool',
         'float': 'bool',
         'numeric': 'bool',
+        'guid': 'object',
     }
     if self.flavor == 'oracle':
         pd_db_df_aliases['int'] = 'numeric'
@@ -2609,7 +2626,6 @@ def get_alter_columns_queries(
     else:
         numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ == 'numeric'])
 
-    pipe_dtypes = pipe.dtypes
     numeric_type = get_db_type_from_pd_type('numeric', self.flavor, as_sqlalchemy=False)
     text_type = get_db_type_from_pd_type('str', self.flavor, as_sqlalchemy=False)
     altered_cols_types = {