meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.1__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -367,6 +367,7 @@ def get_pipe_data(
 
     from meerschaum.utils.packages import import_pandas
     from meerschaum.utils.dataframe import parse_df_datetimes, add_missing_cols_to_df
+    from meerschaum.utils.dtypes import are_dtypes_equal
     pd = import_pandas()
     try:
         df = pd.read_json(StringIO(response.text))
@@ -382,16 +383,17 @@ def get_pipe_data(
         ignore_cols = [
             col
             for col, dtype in pipe.dtypes.items()
-            if 'datetime' not in str(dtype)
+            if not are_dtypes_equal(str(dtype), 'datetime')
         ],
-        debug = debug,
+        strip_timezone=(pipe.tzinfo is None),
+        debug=debug,
     )
     return df
 
 
 def get_pipe_id(
     self,
-    pipe: meerschuam.Pipe,
+    pipe: mrsm.Pipe,
     debug: bool = False,
 ) -> int:
     """Get a Pipe's ID from the API."""
@@ -413,7 +415,7 @@ def get_pipe_id(
 
 def get_pipe_attributes(
     self,
-    pipe: meerschaum.Pipe,
+    pipe: mrsm.Pipe,
     debug: bool = False,
 ) -> Dict[str, Any]:
     """Get a Pipe's attributes from the API
@@ -439,7 +441,7 @@ def get_pipe_attributes(
 
 def get_sync_time(
     self,
-    pipe: 'meerschaum.Pipe',
+    pipe: mrsm.Pipe,
     params: Optional[Dict[str, Any]] = None,
     newest: bool = True,
     debug: bool = False,
@@ -669,11 +671,11 @@ def clear_pipe(
     kw.pop('force', None)
     return self.do_action_legacy(
         ['clear', 'pipes'],
-        connector_keys = pipe.connector_keys,
-        metric_keys = pipe.metric_key,
-        location_keys = pipe.location_key,
-        force = True,
-        debug = debug,
+        connector_keys=pipe.connector_keys,
+        metric_keys=pipe.metric_key,
+        location_keys=pipe.location_key,
+        force=True,
+        debug=debug,
         **kw
     )
 
@@ -690,7 +692,7 @@ def get_pipe_columns_types(
     ----------
     pipe: meerschaum.Pipe
         The pipe whose columns to be queried.
-        
+
     Returns
     -------
     A dictionary mapping column names to their database types.
@@ -707,11 +709,42 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug = debug
+        debug=debug
+    )
+    j = response.json()
+    if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
+        warn(j['detail'])
+        return None
+    if not isinstance(j, dict):
+        warn(response.text)
+        return None
+    return j
+
+
+def get_pipe_columns_indices(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
+    """
+    Fetch the index information for a pipe.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose columns to be queried.
+
+    Returns
+    -------
+    A dictionary mapping column names to a list of associated index information.
+    """
+    r_url = pipe_r_url(pipe) + '/columns/indices'
+    response = self.get(
+        r_url,
+        debug=debug
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
-        from meerschaum.utils.warnings import warn
         warn(j['detail'])
         return None
     if not isinstance(j, dict):

meerschaum/connectors/sql/_SQLConnector.py

@@ -67,6 +67,8 @@ class SQLConnector(Connector):
         get_pipe_columns_types,
         get_to_sql_dtype,
         get_pipe_schema,
+        create_pipe_table_from_df,
+        get_pipe_columns_indices,
     )
     from ._plugins import (
         register_plugin,
@@ -141,9 +143,9 @@ class SQLConnector(Connector):
             if uri.startswith('postgres') and not uri.startswith('postgresql'):
                 uri = uri.replace('postgres', 'postgresql', 1)
             if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
-                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
+                uri = uri.replace('postgresql://', 'postgresql+psycopg://', 1)
             if uri.startswith('timescaledb://'):
-                uri = uri.replace('timescaledb://', 'postgresql://', 1)
+                uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
@@ -155,7 +157,6 @@ class SQLConnector(Connector):
             if flavor:
                 kw['flavor'] = flavor
 
-
         ### set __dict__ in base class
         super().__init__(
             'sql',

meerschaum/connectors/sql/_fetch.py

@@ -90,16 +90,18 @@ def fetch(
     )
     ### if sqlite, parse for datetimes
     if not as_hook_results and self.flavor == 'sqlite':
-        from meerschaum.utils.misc import parse_df_datetimes
+        from meerschaum.utils.dataframe import parse_df_datetimes
+        from meerschaum.utils.dtypes import are_dtypes_equal
         ignore_cols = [
             col
             for col, dtype in pipe.dtypes.items()
-            if 'datetime' not in str(dtype)
+            if not are_dtypes_equal(str(dtype), 'datetime')
         ]
         return (
             parse_df_datetimes(
                 chunk,
                 ignore_cols=ignore_cols,
+                strip_timezone=(pipe.tzinfo is None),
                 debug=debug,
             )
             for chunk in chunks