meerschaum 2.8.4__py3-none-any.whl → 2.9.0__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -731,7 +731,7 @@ def get_create_index_queries(
                         ) + f"{primary_key_name})"
                     ),
                 ])
-            elif self.flavor in ('citus', 'postgresql', 'duckdb'):
+            elif self.flavor in ('citus', 'postgresql', 'duckdb', 'postgis'):
                 primary_queries.extend([
                     (
                         f"ALTER TABLE {_pipe_name}\n"
@@ -1052,6 +1052,7 @@ def get_pipe_data(
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
+        attempt_cast_to_geometry,
         are_dtypes_equal,
     )
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
@@ -1138,6 +1139,11 @@ def get_pipe_data(
         for col, typ in pipe.dtypes.items()
         if typ == 'bytes' and col in dtypes
     ]
+    geometry_columns = [
+        col
+        for col, typ in pipe.dtypes.items()
+        if typ.startswith('geometry') and col in dtypes
+    ]
 
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
 
@@ -1162,6 +1168,11 @@ def get_pipe_data(
             continue
         df[col] = df[col].apply(attempt_cast_to_bytes)
 
+    for col in geometry_columns:
+        if col not in df.columns:
+            continue
+        df[col] = df[col].apply(attempt_cast_to_geometry)
+
     if self.flavor == 'sqlite':
         ignore_dt_cols = [
             col
@@ -1511,7 +1522,7 @@ def get_pipe_attributes(
             if isinstance(parameters, str) and parameters[0] == '{':
                 parameters = json.loads(parameters)
             attributes['parameters'] = parameters
-        except Exception as e:
+        except Exception:
             attributes['parameters'] = {}
 
     return attributes
@@ -1533,7 +1544,11 @@ def create_pipe_table_from_df(
         get_datetime_cols,
         get_bytes_cols,
     )
-    from meerschaum.utils.sql import get_create_table_queries, sql_item_name
+    from meerschaum.utils.sql import (
+        get_create_table_queries,
+        sql_item_name,
+        get_create_schema_if_not_exists_queries,
+    )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
@@ -1590,15 +1605,21 @@ def create_pipe_table_from_df(
     if autoincrement:
         _ = new_dtypes.pop(primary_key, None)
 
+    schema = self.get_pipe_schema(pipe)
     create_table_queries = get_create_table_queries(
         new_dtypes,
         pipe.target,
         self.flavor,
-        schema=self.get_pipe_schema(pipe),
+        schema=schema,
         primary_key=primary_key,
         primary_key_db_type=primary_key_db_type,
         datetime_column=dt_col,
     )
+    if schema:
+        create_table_queries = (
+            get_create_schema_if_not_exists_queries(schema, self.flavor)
+            + create_table_queries
+        )
     success = all(
         self.exec_queries(create_table_queries, break_on_error=True, rollback=True, debug=debug)
     )
@@ -2074,6 +2095,7 @@ def sync_pipe_inplace(
         get_update_queries,
         get_null_replacement,
         get_create_table_queries,
+        get_create_schema_if_not_exists_queries,
         get_table_cols_types,
         session_execute,
         dateadd_str,
@@ -2153,18 +2175,28 @@ def sync_pipe_inplace(
             warn(drop_stale_msg)
         return drop_stale_success, drop_stale_msg
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import(
+        'sqlalchemy',
+        'sqlalchemy.orm',
+    )
     if not pipe.exists(debug=debug):
+        schema = self.get_pipe_schema(pipe)
         create_pipe_queries = get_create_table_queries(
             metadef,
             pipe.target,
             self.flavor,
-            schema=self.get_pipe_schema(pipe),
+            schema=schema,
             primary_key=primary_key,
             primary_key_db_type=primary_key_db_type,
             autoincrement=autoincrement,
             datetime_column=dt_col,
         )
+        if schema:
+            create_pipe_queries = (
+                get_create_schema_if_not_exists_queries(schema, self.flavor)
+                + create_pipe_queries
+            )
+
         results = self.exec_queries(create_pipe_queries, debug=debug)
         if not all(results):
             _ = clean_up_temp_tables()

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,7 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -798,6 +798,7 @@ def to_sql(
         get_numeric_cols,
         get_uuid_cols,
         get_bytes_cols,
+        get_geometry_cols,
     )
     from meerschaum.utils.dtypes import (
         are_dtypes_equal,
@@ -805,7 +806,9 @@ def to_sql(
         encode_bytes_for_bytea,
         serialize_bytes,
         serialize_decimal,
+        serialize_geometry,
         json_serialize_value,
+        get_geometry_type_srid,
     )
     from meerschaum.utils.dtypes.sql import (
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
@@ -822,6 +825,7 @@ def to_sql(
 
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
+    geometry_cols = get_geometry_cols(df)
     ### NOTE: This excludes non-numeric serialized Decimals (e.g. SQLite).
     numeric_cols_dtypes = {
         col: typ
@@ -830,7 +834,6 @@ def to_sql(
             col in df.columns
             and 'numeric' in str(typ).lower()
         )
-        
     }
     numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
     numeric_cols_precisions_scales = {
@@ -841,6 +844,22 @@ def to_sql(
         )
         for col, typ in numeric_cols_dtypes.items()
     }
+    geometry_cols_dtypes = {
+        col: typ
+        for col, typ in kw.get('dtype', {}).items()
+        if (
+            col in df.columns
+            and 'geometry' in str(typ).lower() or 'geography' in str(typ).lower()
+        )
+    }
+    geometry_cols.extend([col for col in geometry_cols_dtypes if col not in geometry_cols])
+    geometry_cols_types_srids = {
+        col: (typ.geometry_type, typ.srid)
+        if hasattr(typ, 'srid')
+        else get_geometry_type_srid()
+        for col, typ in geometry_cols_dtypes.items()
+    }
+
     cols_pd_types = {
         col: get_pd_type_from_db_type(str(typ))
         for col, typ in kw.get('dtype', {}).items()
@@ -856,8 +875,9 @@ def to_sql(
     }
 
     enable_bulk_insert = mrsm.get_config(
-        'system', 'connectors', 'sql', 'bulk_insert'
-    ).get(self.flavor, False)
+        'system', 'connectors', 'sql', 'bulk_insert', self.flavor,
+        warn=False,
+    ) or False
     stats = {'target': name}
     ### resort to defaults if None
     copied = False
@@ -901,6 +921,17 @@ def to_sql(
             )
         )
 
+    for col in geometry_cols:
+        geometry_type, srid = geometry_cols_types_srids.get(col, get_geometry_type_srid())
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            df[col] = df[col].apply(
+                functools.partial(
+                    serialize_geometry,
+                    as_wkt=(self.flavor == 'mssql')
+                )
+            )
+
     stats['method'] = method.__name__ if hasattr(method, '__name__') else str(method)
 
     default_chunksize = self._sys_config.get('chunksize', None)

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -50,6 +50,8 @@ class ValkeyConnector(Connector):
         get_sync_time,
         get_pipe_rowcount,
         fetch_pipes_keys,
+        get_document_key,
+        get_table_quoted_doc_key,
     )
     from ._fetch import (
         fetch,

meerschaum/connectors/valkey/_pipes.py

@@ -10,8 +10,9 @@ from datetime import datetime, timezone
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, Union, Optional, Dict, List, Tuple
-from meerschaum.utils.misc import json_serialize_datetime, string_to_dict
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.misc import string_to_dict
+from meerschaum.utils.dtypes import json_serialize_value
+from meerschaum.utils.warnings import warn, dprint
 from meerschaum.config.static import STATIC_CONFIG
 
 PIPES_TABLE: str = 'mrsm_pipes'
@@ -46,25 +47,15 @@ def serialize_document(doc: Dict[str, Any]) -> str:
     -------
     A serialized string for the document.
     """
-    from meerschaum.utils.dtypes import serialize_bytes
     return json.dumps(
         doc,
-        default=(
-            lambda x: (
-                json_serialize_datetime(x)
-                if hasattr(x, 'tzinfo')
-                else (
-                    serialize_bytes(x)
-                    if isinstance(x, bytes)
-                    else str(x)
-                )
-            )
-        ),
+        default=json_serialize_value,
         separators=(',', ':'),
         sort_keys=True,
     )
 
 
+@staticmethod
 def get_document_key(
     doc: Dict[str, Any],
     indices: List[str],
@@ -91,25 +82,39 @@ def get_document_key(
     from meerschaum.utils.dtypes import coerce_timezone
     index_vals = {
         key: (
-            str(val)
+            str(val).replace(':', COLON)
             if not isinstance(val, datetime)
             else str(int(coerce_timezone(val).replace(tzinfo=timezone.utc).timestamp()))
         )
         for key, val in doc.items()
-        if key in indices
-    } if indices else {}
-    indices_str = ((table_name + ':indices:') if table_name else '') + ','.join(
-        sorted(
-            [
-                f'{key}{COLON}{val}'
-                for key, val in index_vals.items()
-            ]
+        if ((key in indices) if indices else True)
+    }
+    indices_str = (
+        (
+            (
+                (
+                    table_name
+                    + ':'
+                    + ('indices:' if True else '')
+                )
+            )
+            if table_name
+            else ''
+        ) + ','.join(
+            sorted(
+                [
+                    f'{key}{COLON}{val}'
+                    for key, val in index_vals.items()
+                ]
+            )
         )
-    ) if indices else serialize_document(doc)
+    )
     return indices_str
 
 
+@classmethod
 def get_table_quoted_doc_key(
+    cls,
     table_name: str,
     doc: Dict[str, Any],
     indices: List[str],
@@ -120,7 +125,7 @@ def get_table_quoted_doc_key(
     """
     return json.dumps(
         {
-            get_document_key(doc, indices, table_name): serialize_document(doc),
+            cls.get_document_key(doc, indices, table_name): serialize_document(doc),
             **(
                 {datetime_column: doc.get(datetime_column, 0)}
                 if datetime_column
@@ -129,7 +134,7 @@ def get_table_quoted_doc_key(
         },
         sort_keys=True,
         separators=(',', ':'),
-        default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+        default=json_serialize_value,
     )
 
 
@@ -377,7 +382,7 @@ def delete_pipe(
         doc = docs[0]
         doc_str = json.dumps(
             doc,
-            default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+            default=json_serialize_value,
             separators=(',', ':'),
             sort_keys=True,
         )
@@ -443,11 +448,16 @@ def get_pipe_data(
             debug=debug,
         )
     ]
+    print(f"{ix_docs=}")
     try:
         docs_strings = [
-            self.get(get_document_key(
-                doc, indices, table_name
-            ))
+            self.get(
+                self.get_document_key(
+                    doc,
+                    indices,
+                    table_name,
+                )
+            )
             for doc in ix_docs
         ]
     except Exception as e:
@@ -535,7 +545,7 @@ def sync_pipe(
     def _serialize_indices_docs(_docs):
         return [
             {
-                'ix': get_document_key(doc, indices),
+                'ix': self.get_document_key(doc, indices),
                 **(
                     {
                         dt_col: doc.get(dt_col, 0)
@@ -594,7 +604,7 @@ def sync_pipe(
     unseen_docs = unseen_df.to_dict(orient='records') if unseen_df is not None else []
     unseen_indices_docs = _serialize_indices_docs(unseen_docs)
     unseen_ix_vals = {
-        get_document_key(doc, indices, table_name): serialize_document(doc)
+        self.get_document_key(doc, indices, table_name): serialize_document(doc)
         for doc in unseen_docs
     }
     for key, val in unseen_ix_vals.items():
@@ -615,7 +625,7 @@ def sync_pipe(
 
     update_docs = update_df.to_dict(orient='records') if update_df is not None else []
     update_ix_docs = {
-        get_document_key(doc, indices, table_name): doc
+        self.get_document_key(doc, indices, table_name): doc
         for doc in update_docs
     }
     existing_docs_data = {
@@ -633,7 +643,7 @@ def sync_pipe(
         if key not in existing_docs
     }
     new_ix_vals = {
-        get_document_key(doc, indices, table_name): serialize_document(doc)
+        self.get_document_key(doc, indices, table_name): serialize_document(doc)
         for doc in new_update_docs.values()
     }
     for key, val in new_ix_vals.items():
@@ -743,8 +753,8 @@ def clear_pipe(
     table_name = self.quote_table(pipe.target)
     indices = [col for col in pipe.columns.values() if col]
     for doc in docs:
-        set_doc_key = get_document_key(doc, indices)
-        table_doc_key = get_document_key(doc, indices, table_name)
+        set_doc_key = self.get_document_key(doc, indices)
+        table_doc_key = self.get_document_key(doc, indices, table_name)
         try:
             if dt_col:
                 self.client.zrem(table_name, set_doc_key)
@@ -826,13 +836,15 @@ def get_pipe_rowcount(
         return 0
 
     try:
-        if begin is None and end is None and params is None:
+        if begin is None and end is None and not params:
             return (
                 self.client.zcard(table_name)
                 if dt_col
-                else self.client.llen(table_name)
+                else self.client.scard(table_name)
             )
-    except Exception:
+    except Exception as e:
+        if debug:
+            dprint(f"Failed to get rowcount for {pipe}:\n{e}")
         return None
 
     df = pipe.get_data(begin=begin, end=end, params=params, debug=debug)

meerschaum/core/Pipe/__init__.py

@@ -137,6 +137,7 @@ class Pipe:
         _persist_new_numeric_columns,
         _persist_new_uuid_columns,
         _persist_new_bytes_columns,
+        _persist_new_geometry_columns,
     )
     from ._verify import (
         verify,

meerschaum/core/Pipe/_data.py

@@ -88,9 +88,8 @@ def get_data(
     limit: Optional[int], default None
         If provided, cap the dataframe to this many rows.
 
-    fresh: bool, default True
+    fresh: bool, default False
         If `True`, skip local cache and directly query the instance connector.
-        Defaults to `True`.
 
     debug: bool, default False
         Verbosity toggle.

meerschaum/core/Pipe/_sync.py

@@ -158,6 +158,7 @@ def sync(
         'error_callback': error_callback,
         'sync_chunks': sync_chunks,
         'chunksize': chunksize,
+        'safe_copy': True,
     })
 
     ### NOTE: Invalidate `_exists` cache before and after syncing.
@@ -268,6 +269,7 @@ def sync(
                             **kw
                         )
                     )
+                    kw['safe_copy'] = False
             except Exception as e:
                 get_console().print_exception(
                     suppress=[
@@ -402,6 +404,7 @@ def sync(
         self._persist_new_numeric_columns(df, debug=debug)
         self._persist_new_uuid_columns(df, debug=debug)
         self._persist_new_bytes_columns(df, debug=debug)
+        self._persist_new_geometry_columns(df, debug=debug)
 
         if debug:
             dprint(
@@ -1009,7 +1012,7 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'numeric' for col in numeric_cols})
+    dtypes.update({col: 'numeric' for col in new_numeric_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
         edit_success, edit_msg = self.edit(interactive=False, debug=debug)
@@ -1034,7 +1037,7 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'uuid' for col in uuid_cols})
+    dtypes.update({col: 'uuid' for col in new_uuid_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
         edit_success, edit_msg = self.edit(interactive=False, debug=debug)
@@ -1059,7 +1062,7 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'json' for col in json_cols})
+    dtypes.update({col: 'json' for col in new_json_cols})
     self.parameters['dtypes'] = dtypes
 
     if not self.temporary:
@@ -1085,7 +1088,64 @@ def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'bytes' for col in bytes_cols})
+    dtypes.update({col: 'bytes' for col in new_bytes_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"
+
+
+def _persist_new_geometry_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new `geometry` columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_geometry_cols
+    geometry_cols_types_srids = get_geometry_cols(df, with_types_srids=True)
+    existing_geometry_cols = [
+        col
+        for col, typ in self.dtypes.items()
+        if typ.startswith('geometry') or typ.startswith('geography')
+    ]
+    new_geometry_cols = [
+        col
+        for col in geometry_cols_types_srids
+        if col not in existing_geometry_cols
+    ]
+    if not new_geometry_cols:
+        return True, "Success"
+
+    self._attributes_sync_time = None
+    dtypes = self.parameters.get('dtypes', {})
+
+    new_cols_types = {}
+    for col, (geometry_type, srid) in geometry_cols_types_srids.items():
+        if col not in new_geometry_cols:
+            continue
+
+        new_dtype = "geometry"
+        modifier = ""
+        if not srid and geometry_type.lower() == 'geometry':
+            new_cols_types[col] = new_dtype
+            continue
+
+        modifier = "["
+        if geometry_type.lower() != 'geometry':
+            modifier += f"{geometry_type}"
+
+        if srid:
+            if modifier != '[':
+                modifier += ", "
+            modifier += f"{srid}"
+        modifier += "]"
+        new_cols_types[col] = f"{new_dtype}{modifier}"
+
+    dtypes.update(new_cols_types)
     self.parameters['dtypes'] = dtypes
 
     if not self.temporary:

meerschaum/plugins/_Plugin.py

@@ -450,7 +450,7 @@ class Plugin:
                 success, msg = False, (
                     f"Failed to run post-install setup for plugin '{self}'." + '\n' +
                     f"Check `setup()` in '{self.__file__}' for more information " +
-                    f"(no error message provided)."
+                    "(no error message provided)."
                 )
             else:
                 success, msg = True, success_msg
@@ -458,7 +458,7 @@ class Plugin:
             success = True
             msg = (
                 f"Post-install for plugin '{self}' returned None. " +
-                f"Assuming plugin successfully installed."
+                "Assuming plugin successfully installed."
             )
             warn(msg)
         else:
@@ -469,7 +469,7 @@ class Plugin:
             )
 
         _ongoing_installations.remove(self.full_name)
-        module = self.module
+        _ = self.module
         return success, msg
 
 
@@ -716,13 +716,14 @@ class Plugin:
         return required
 
 
-    def get_required_plugins(self, debug: bool=False) -> List[meerschaum.plugins.Plugin]:
+    def get_required_plugins(self, debug: bool=False) -> List[mrsm.plugins.Plugin]:
         """
         Return a list of required Plugin objects.
         """
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
         from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum.connectors.parse import is_valid_connector_keys
         plugins = []
         _deps = self.get_dependencies(debug=debug)
         sep = STATIC_CONFIG['plugins']['repo_separator']
@@ -731,11 +732,13 @@ class Plugin:
             if _d.startswith('plugin:') and len(_d) > len('plugin:')
         ]
         default_repo_keys = get_config('meerschaum', 'default_repository')
+        skipped_repo_keys = set()
+
         for _plugin_name in plugin_names:
             if sep in _plugin_name:
                 try:
                     _plugin_name, _repo_keys = _plugin_name.split(sep)
-                except Exception as e:
+                except Exception:
                     _repo_keys = default_repo_keys
                     warn(
                         f"Invalid repo keys for required plugin '{_plugin_name}'.\n    "
@@ -744,7 +747,20 @@ class Plugin:
                     )
             else:
                 _repo_keys = default_repo_keys
+
+            if _repo_keys in skipped_repo_keys:
+                continue
+
+            if not is_valid_connector_keys(_repo_keys):
+                warn(
+                    f"Invalid connector '{_repo_keys}'.\n"
+                    f"    Skipping required plugins from repository '{_repo_keys}'",
+                    stack=False,
+                )
+                continue
+
             plugins.append(Plugin(_plugin_name, repo=_repo_keys))
+
         return plugins