meerschaum 2.9.0rc2__py3-none-any.whl → 2.9.1__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -414,19 +414,24 @@ def get_pipe_id(
     self,
     pipe: mrsm.Pipe,
     debug: bool = False,
-) -> int:
+) -> Union[int, str, None]:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/id',
-        debug = debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(f"Got pipe ID: {response.text}")
     try:
         if is_int(response.text):
             return int(response.text)
+        if response.text and response.text[0] != '{':
+            return response.text
     except Exception as e:
         warn(f"Failed to get the ID for {pipe}:\n{e}")
     return None
@@ -450,7 +455,13 @@ def get_pipe_attributes(
     If the pipe does not exist, return an empty dictionary.
     """
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/attributes', debug=debug)
+    response = self.get(
+        r_url + '/attributes',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug
+    )
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -489,9 +500,13 @@ def get_sync_time(
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
-        json = params,
-        params = {'newest': newest, 'debug': debug},
-        debug = debug,
+        json=params,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+            'newest': newest,
+            'debug': debug,
+        },
+        debug=debug,
     )
     if not response:
         warn(f"Failed to get the sync time for {pipe}:\n" + response.text)
@@ -532,7 +547,13 @@ def pipe_exists(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/exists', debug=debug)
+    response = self.get(
+        r_url + '/exists',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
+    )
     if not response:
         warn(f"Failed to check if {pipe} exists:\n{response.text}")
         return False
@@ -570,8 +591,8 @@ def create_metadata(
 def get_pipe_rowcount(
     self,
     pipe: mrsm.Pipe,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
     params: Optional[Dict[str, Any]] = None,
     remote: bool = False,
     debug: bool = False,
@@ -583,10 +604,10 @@ def get_pipe_rowcount(
     pipe: 'meerschaum.Pipe':
         The pipe whose row count we are counting.
         
-    begin: Optional[datetime], default None
+    begin: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
-    end: Optional[datetime]
+    end: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
     params: Optional[Dict[str, Any]], default None
@@ -608,6 +629,7 @@ def get_pipe_rowcount(
             'begin': begin,
             'end': end,
             'remote': remote,
+            'instance': self.get_pipe_instance_keys(pipe),
         },
         debug = debug
     )
@@ -645,7 +667,10 @@ def drop_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/drop',
-        debug = debug,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -668,6 +693,9 @@ def drop_pipe(
 def clear_pipe(
     self,
     pipe: mrsm.Pipe,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -683,20 +711,33 @@ def clear_pipe(
     -------
     A success tuple.
     """
-    kw.pop('metric_keys', None)
-    kw.pop('connector_keys', None)
-    kw.pop('location_keys', None)
-    kw.pop('action', None)
-    kw.pop('force', None)
-    return self.do_action_legacy(
-        ['clear', 'pipes'],
-        connector_keys=pipe.connector_keys,
-        metric_keys=pipe.metric_key,
-        location_keys=pipe.location_key,
-        force=True,
+    r_url = pipe_r_url(pipe)
+    response = self.delete(
+        r_url + '/clear',
+        params={
+            'begin': begin,
+            'end': end,
+            'params': json.dumps(params),
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
         debug=debug,
-        **kw
     )
+    if debug:
+        dprint(response.text)
+
+    try:
+        data = response.json()
+    except Exception as e:
+        return False, f"Failed to clear {pipe} with constraints {begin=}, {end=}, {params=}."
+
+    if isinstance(data, list):
+        response_tuple = data[0], data[1]
+    elif 'detail' in response.json():
+        response_tuple = response.__bool__(), data['detail']
+    else:
+        response_tuple = response.__bool__(), response.text
+
+    return response_tuple
 
 
 def get_pipe_columns_types(
@@ -728,7 +769,10 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -760,7 +804,10 @@ def get_pipe_columns_indices(
     r_url = pipe_r_url(pipe) + '/columns/indices'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -779,14 +826,16 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str
     r_url = pipe_r_url(pipe) + '/indices/names'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
         warn(j['detail'])
-        return None
+        return {}
     if not isinstance(j, dict):
         warn(response.text)
-        return None
+        return {}
     return j
-

meerschaum/connectors/sql/_pipes.py

@@ -499,6 +499,7 @@ def get_create_index_queries(
         get_rename_table_queries,
         COALESCE_UNIQUE_INDEX_FLAVORS,
     )
+    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
         get_db_type_from_pd_type,
         get_pd_type_from_db_type,
@@ -793,8 +794,19 @@ def get_create_index_queries(
         cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
         if not cols_names:
             continue
+
         cols_names_str = ", ".join(cols_names)
-        index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
+        index_query_params_clause = f" ({cols_names_str})"
+        if self.flavor == 'postgis':
+            for col in cols:
+                col_typ = existing_cols_pd_types.get(cols[0], 'object')
+                if col_typ != 'object' and are_dtypes_equal(col_typ, 'geometry'):
+                    index_query_params_clause = f" USING GIST ({cols_names_str})"
+                    break
+
+        index_queries[ix_key] = [
+            f"CREATE INDEX {ix_name} ON {_pipe_name}{index_query_params_clause}"
+        ]
 
     indices_cols_str = ', '.join(
         list({
@@ -1544,7 +1556,11 @@ def create_pipe_table_from_df(
         get_datetime_cols,
         get_bytes_cols,
     )
-    from meerschaum.utils.sql import get_create_table_queries, sql_item_name
+    from meerschaum.utils.sql import (
+        get_create_table_queries,
+        sql_item_name,
+        get_create_schema_if_not_exists_queries,
+    )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
@@ -1601,15 +1617,21 @@ def create_pipe_table_from_df(
     if autoincrement:
         _ = new_dtypes.pop(primary_key, None)
 
+    schema = self.get_pipe_schema(pipe)
     create_table_queries = get_create_table_queries(
         new_dtypes,
         pipe.target,
         self.flavor,
-        schema=self.get_pipe_schema(pipe),
+        schema=schema,
         primary_key=primary_key,
         primary_key_db_type=primary_key_db_type,
         datetime_column=dt_col,
     )
+    if schema:
+        create_table_queries = (
+            get_create_schema_if_not_exists_queries(schema, self.flavor)
+            + create_table_queries
+        )
     success = all(
         self.exec_queries(create_table_queries, break_on_error=True, rollback=True, debug=debug)
     )
@@ -2085,6 +2107,7 @@ def sync_pipe_inplace(
         get_update_queries,
         get_null_replacement,
         get_create_table_queries,
+        get_create_schema_if_not_exists_queries,
         get_table_cols_types,
         session_execute,
         dateadd_str,
@@ -2164,18 +2187,28 @@ def sync_pipe_inplace(
             warn(drop_stale_msg)
         return drop_stale_success, drop_stale_msg
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import(
+        'sqlalchemy',
+        'sqlalchemy.orm',
+    )
     if not pipe.exists(debug=debug):
+        schema = self.get_pipe_schema(pipe)
         create_pipe_queries = get_create_table_queries(
             metadef,
             pipe.target,
             self.flavor,
-            schema=self.get_pipe_schema(pipe),
+            schema=schema,
             primary_key=primary_key,
             primary_key_db_type=primary_key_db_type,
             autoincrement=autoincrement,
             datetime_column=dt_col,
         )
+        if schema:
+            create_pipe_queries = (
+                get_create_schema_if_not_exists_queries(schema, self.flavor)
+                + create_pipe_queries
+            )
+
         results = self.exec_queries(create_pipe_queries, debug=debug)
         if not all(results):
             _ = clean_up_temp_tables()

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -50,6 +50,8 @@ class ValkeyConnector(Connector):
         get_sync_time,
         get_pipe_rowcount,
         fetch_pipes_keys,
+        get_document_key,
+        get_table_quoted_doc_key,
     )
     from ._fetch import (
         fetch,

meerschaum/connectors/valkey/_pipes.py

@@ -10,8 +10,9 @@ from datetime import datetime, timezone
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, Union, Optional, Dict, List, Tuple
-from meerschaum.utils.misc import json_serialize_datetime, string_to_dict
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.misc import string_to_dict
+from meerschaum.utils.dtypes import json_serialize_value
+from meerschaum.utils.warnings import warn, dprint
 from meerschaum.config.static import STATIC_CONFIG
 
 PIPES_TABLE: str = 'mrsm_pipes'
@@ -46,25 +47,15 @@ def serialize_document(doc: Dict[str, Any]) -> str:
     -------
     A serialized string for the document.
     """
-    from meerschaum.utils.dtypes import serialize_bytes
     return json.dumps(
         doc,
-        default=(
-            lambda x: (
-                json_serialize_datetime(x)
-                if hasattr(x, 'tzinfo')
-                else (
-                    serialize_bytes(x)
-                    if isinstance(x, bytes)
-                    else str(x)
-                )
-            )
-        ),
+        default=json_serialize_value,
         separators=(',', ':'),
         sort_keys=True,
     )
 
 
+@staticmethod
 def get_document_key(
     doc: Dict[str, Any],
     indices: List[str],
@@ -91,25 +82,39 @@ def get_document_key(
     from meerschaum.utils.dtypes import coerce_timezone
     index_vals = {
         key: (
-            str(val)
+            str(val).replace(':', COLON)
             if not isinstance(val, datetime)
             else str(int(coerce_timezone(val).replace(tzinfo=timezone.utc).timestamp()))
         )
         for key, val in doc.items()
-        if key in indices
-    } if indices else {}
-    indices_str = ((table_name + ':indices:') if table_name else '') + ','.join(
-        sorted(
-            [
-                f'{key}{COLON}{val}'
-                for key, val in index_vals.items()
-            ]
+        if ((key in indices) if indices else True)
+    }
+    indices_str = (
+        (
+            (
+                (
+                    table_name
+                    + ':'
+                    + ('indices:' if True else '')
+                )
+            )
+            if table_name
+            else ''
+        ) + ','.join(
+            sorted(
+                [
+                    f'{key}{COLON}{val}'
+                    for key, val in index_vals.items()
+                ]
+            )
         )
-    ) if indices else serialize_document(doc)
+    )
     return indices_str
 
 
+@classmethod
 def get_table_quoted_doc_key(
+    cls,
     table_name: str,
     doc: Dict[str, Any],
     indices: List[str],
@@ -120,7 +125,7 @@ def get_table_quoted_doc_key(
     """
     return json.dumps(
         {
-            get_document_key(doc, indices, table_name): serialize_document(doc),
+            cls.get_document_key(doc, indices, table_name): serialize_document(doc),
             **(
                 {datetime_column: doc.get(datetime_column, 0)}
                 if datetime_column
@@ -129,7 +134,7 @@ def get_table_quoted_doc_key(
         },
         sort_keys=True,
         separators=(',', ':'),
-        default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+        default=json_serialize_value,
     )
 
 
@@ -377,7 +382,7 @@ def delete_pipe(
         doc = docs[0]
         doc_str = json.dumps(
             doc,
-            default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+            default=json_serialize_value,
             separators=(',', ':'),
             sort_keys=True,
         )
@@ -443,11 +448,16 @@ def get_pipe_data(
             debug=debug,
         )
     ]
+    print(f"{ix_docs=}")
     try:
         docs_strings = [
-            self.get(get_document_key(
-                doc, indices, table_name
-            ))
+            self.get(
+                self.get_document_key(
+                    doc,
+                    indices,
+                    table_name,
+                )
+            )
             for doc in ix_docs
         ]
     except Exception as e:
@@ -535,7 +545,7 @@ def sync_pipe(
     def _serialize_indices_docs(_docs):
         return [
             {
-                'ix': get_document_key(doc, indices),
+                'ix': self.get_document_key(doc, indices),
                 **(
                     {
                         dt_col: doc.get(dt_col, 0)
@@ -594,7 +604,7 @@ def sync_pipe(
     unseen_docs = unseen_df.to_dict(orient='records') if unseen_df is not None else []
     unseen_indices_docs = _serialize_indices_docs(unseen_docs)
     unseen_ix_vals = {
-        get_document_key(doc, indices, table_name): serialize_document(doc)
+        self.get_document_key(doc, indices, table_name): serialize_document(doc)
         for doc in unseen_docs
     }
     for key, val in unseen_ix_vals.items():
@@ -615,7 +625,7 @@ def sync_pipe(
 
     update_docs = update_df.to_dict(orient='records') if update_df is not None else []
     update_ix_docs = {
-        get_document_key(doc, indices, table_name): doc
+        self.get_document_key(doc, indices, table_name): doc
         for doc in update_docs
     }
     existing_docs_data = {
@@ -633,7 +643,7 @@ def sync_pipe(
         if key not in existing_docs
     }
     new_ix_vals = {
-        get_document_key(doc, indices, table_name): serialize_document(doc)
+        self.get_document_key(doc, indices, table_name): serialize_document(doc)
         for doc in new_update_docs.values()
     }
     for key, val in new_ix_vals.items():
@@ -743,8 +753,8 @@ def clear_pipe(
     table_name = self.quote_table(pipe.target)
     indices = [col for col in pipe.columns.values() if col]
     for doc in docs:
-        set_doc_key = get_document_key(doc, indices)
-        table_doc_key = get_document_key(doc, indices, table_name)
+        set_doc_key = self.get_document_key(doc, indices)
+        table_doc_key = self.get_document_key(doc, indices, table_name)
         try:
             if dt_col:
                 self.client.zrem(table_name, set_doc_key)
@@ -826,13 +836,15 @@ def get_pipe_rowcount(
         return 0
 
     try:
-        if begin is None and end is None and params is None:
+        if begin is None and end is None and not params:
             return (
                 self.client.zcard(table_name)
                 if dt_col
-                else self.client.llen(table_name)
+                else self.client.scard(table_name)
             )
-    except Exception:
+    except Exception as e:
+        if debug:
+            dprint(f"Failed to get rowcount for {pipe}:\n{e}")
         return None
 
     df = pipe.get_data(begin=begin, end=end, params=params, debug=debug)

meerschaum/core/Pipe/__init__.py

@@ -137,6 +137,7 @@ class Pipe:
         _persist_new_numeric_columns,
         _persist_new_uuid_columns,
         _persist_new_bytes_columns,
+        _persist_new_geometry_columns,
     )
     from ._verify import (
         verify,

meerschaum/core/Pipe/_sync.py

@@ -158,6 +158,7 @@ def sync(
         'error_callback': error_callback,
         'sync_chunks': sync_chunks,
         'chunksize': chunksize,
+        'safe_copy': True,
     })
 
     ### NOTE: Invalidate `_exists` cache before and after syncing.
@@ -268,6 +269,7 @@ def sync(
                             **kw
                         )
                     )
+                    kw['safe_copy'] = False
             except Exception as e:
                 get_console().print_exception(
                     suppress=[
@@ -402,6 +404,7 @@ def sync(
         self._persist_new_numeric_columns(df, debug=debug)
         self._persist_new_uuid_columns(df, debug=debug)
         self._persist_new_bytes_columns(df, debug=debug)
+        self._persist_new_geometry_columns(df, debug=debug)
 
         if debug:
             dprint(
@@ -1009,7 +1012,7 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'numeric' for col in numeric_cols})
+    dtypes.update({col: 'numeric' for col in new_numeric_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
         edit_success, edit_msg = self.edit(interactive=False, debug=debug)
@@ -1034,7 +1037,7 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'uuid' for col in uuid_cols})
+    dtypes.update({col: 'uuid' for col in new_uuid_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
         edit_success, edit_msg = self.edit(interactive=False, debug=debug)
@@ -1059,7 +1062,7 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'json' for col in json_cols})
+    dtypes.update({col: 'json' for col in new_json_cols})
     self.parameters['dtypes'] = dtypes
 
     if not self.temporary:
@@ -1085,7 +1088,64 @@ def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
 
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
-    dtypes.update({col: 'bytes' for col in bytes_cols})
+    dtypes.update({col: 'bytes' for col in new_bytes_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"
+
+
+def _persist_new_geometry_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new `geometry` columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_geometry_cols
+    geometry_cols_types_srids = get_geometry_cols(df, with_types_srids=True)
+    existing_geometry_cols = [
+        col
+        for col, typ in self.dtypes.items()
+        if typ.startswith('geometry') or typ.startswith('geography')
+    ]
+    new_geometry_cols = [
+        col
+        for col in geometry_cols_types_srids
+        if col not in existing_geometry_cols
+    ]
+    if not new_geometry_cols:
+        return True, "Success"
+
+    self._attributes_sync_time = None
+    dtypes = self.parameters.get('dtypes', {})
+
+    new_cols_types = {}
+    for col, (geometry_type, srid) in geometry_cols_types_srids.items():
+        if col not in new_geometry_cols:
+            continue
+
+        new_dtype = "geometry"
+        modifier = ""
+        if not srid and geometry_type.lower() == 'geometry':
+            new_cols_types[col] = new_dtype
+            continue
+
+        modifier = "["
+        if geometry_type.lower() != 'geometry':
+            modifier += f"{geometry_type}"
+
+        if srid:
+            if modifier != '[':
+                modifier += ", "
+            modifier += f"{srid}"
+        modifier += "]"
+        new_cols_types[col] = f"{new_dtype}{modifier}"
+
+    dtypes.update(new_cols_types)
     self.parameters['dtypes'] = dtypes
 
     if not self.temporary: