meerschaum 2.9.0.dev1__tar.gz → 2.9.0rc1__tar.gz

{meerschaum-2.9.0.dev1/meerschaum.egg-info → meerschaum-2.9.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.9.0.dev1
+Version: 2.9.0rc1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -127,12 +127,14 @@ Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
 Requires-Dist: geopandas>=1.0.1; extra == "sql"
+Requires-Dist: shapely>=2.0.7; extra == "sql"
 Requires-Dist: pyarrow>=16.1.0; extra == "sql"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "sql"
 Requires-Dist: partd>=1.4.2; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "sql"
 Requires-Dist: databases>=0.4.0; extra == "sql"
 Requires-Dist: aiosqlite>=0.16.0; extra == "sql"
 Requires-Dist: asyncpg>=0.21.0; extra == "sql"
@@ -188,12 +190,14 @@ Requires-Dist: valkey>=6.0.0; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
 Requires-Dist: geopandas>=1.0.1; extra == "api"
+Requires-Dist: shapely>=2.0.7; extra == "api"
 Requires-Dist: pyarrow>=16.1.0; extra == "api"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "api"
 Requires-Dist: partd>=1.4.2; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "api"
 Requires-Dist: databases>=0.4.0; extra == "api"
 Requires-Dist: aiosqlite>=0.16.0; extra == "api"
 Requires-Dist: asyncpg>=0.21.0; extra == "api"
@@ -294,12 +298,14 @@ Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
 Requires-Dist: geopandas>=1.0.1; extra == "full"
+Requires-Dist: shapely>=2.0.7; extra == "full"
 Requires-Dist: pyarrow>=16.1.0; extra == "full"
 Requires-Dist: dask[complete]>=2024.12.1; extra == "full"
 Requires-Dist: partd>=1.4.2; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "full"
 Requires-Dist: databases>=0.4.0; extra == "full"
 Requires-Dist: aiosqlite>=0.16.0; extra == "full"
 Requires-Dist: asyncpg>=0.21.0; extra == "full"

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/config/_default.py

@@ -70,6 +70,7 @@ default_system_config = {
         'sql': {
             'bulk_insert': {
                 'postgresql': True,
+                'postgis': True,
                 'citus': True,
                 'timescaledb': True,
                 'mssql': True,

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.9.0.dev1"
+__version__ = "2.9.0rc1"

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/connectors/sql/_SQLConnector.py

@@ -151,6 +151,9 @@ class SQLConnector(Connector):
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
+            if uri.startswith('postgis://'):
+                uri = uri.replace('postgis://', 'postgresql+psycopg://', 1)
+                flavor = 'postgis'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
             label = label or from_uri_params.get('label', None)

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/connectors/sql/_cli.py

@@ -15,6 +15,7 @@ from meerschaum.utils.typing import SuccessTuple
 
 flavor_clis = {
     'postgresql'  : 'pgcli',
+    'postgis'  : 'pgcli',
     'timescaledb' : 'pgcli',
     'cockroachdb' : 'pgcli',
     'citus'       : 'pgcli',

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/connectors/sql/_create_engine.py

@@ -7,6 +7,7 @@ This module contains the logic that builds the sqlalchemy engine string.
 """
 
 import traceback
+import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 
 ### determine driver and requirements from flavor
@@ -47,6 +48,16 @@ flavor_configs = {
             'port': 5432,
         },
     },
+    'postgis': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
+        'omit_create_engine': {'method',},
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
+        },
+    },
     'citus': {
         'engine': 'postgresql+psycopg',
         'create_engine': default_create_engine_args,
@@ -162,6 +173,7 @@ install_flavor_drivers = {
     'mariadb': ['pymysql'],
     'timescaledb': ['psycopg'],
     'postgresql': ['psycopg'],
+    'postgis': ['psycopg', 'geoalchemy'],
     'citus': ['psycopg'],
     'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
@@ -198,8 +210,7 @@ def create_engine(
             warn=False,
         )
         if self.flavor == 'mssql':
-            pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
-            pyodbc.pooling = False
+            _init_mssql_sqlalchemy()
     if self.flavor in require_patching_flavors:
         from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
         import pathlib
@@ -257,8 +268,8 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in _uri:
-            if self.flavor == 'timescaledb':
-                engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
+            if self.flavor in ('timescaledb', 'postgis'):
+                engine_str = engine_str.replace(self.flavor, 'postgresql', 1)
             elif _uri.startswith('postgresql://'):
                 engine_str = engine_str.replace('postgresql://', 'postgresql+psycopg2://')
 
@@ -313,3 +324,39 @@ def create_engine(
     if include_uri:
         return engine, engine_str
     return engine
+
+
+def _init_mssql_sqlalchemy():
+    """
+    When first instantiating a SQLAlchemy connection to MSSQL,
+    monkey-patch `pyodbc` handling in SQLAlchemy.
+    """
+    pyodbc, sqlalchemy_dialects_mssql_pyodbc = mrsm.attempt_import(
+        'pyodbc',
+        'sqlalchemy.dialects.mssql.pyodbc',
+        lazy=False,
+        warn=False,
+    )
+    pyodbc.pooling = False
+
+    MSDialect_pyodbc = sqlalchemy_dialects_mssql_pyodbc.MSDialect_pyodbc
+
+    def _handle_geometry(val):
+        from binascii import hexlify
+        hex_str = f"0x{hexlify(val).decode().upper()}"
+        return hex_str
+
+    def custom_on_connect(self):
+        super_ = super(MSDialect_pyodbc, self).on_connect()
+
+        def _on_connect(conn):
+            if super_ is not None:
+                super_(conn)
+
+            self._setup_timestampoffset_type(conn)
+            conn.add_output_converter(-151, _handle_geometry)
+
+        return _on_connect
+
+    ### TODO: Parse proprietary MSSQL geometry bytes into WKB.
+    #  MSDialect_pyodbc.on_connect = custom_on_connect

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/connectors/sql/_pipes.py

@@ -731,7 +731,7 @@ def get_create_index_queries(
                         ) + f"{primary_key_name})"
                     ),
                 ])
-            elif self.flavor in ('citus', 'postgresql', 'duckdb'):
+            elif self.flavor in ('citus', 'postgresql', 'duckdb', 'postgis'):
                 primary_queries.extend([
                     (
                         f"ALTER TABLE {_pipe_name}\n"
@@ -1052,6 +1052,7 @@ def get_pipe_data(
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
+        attempt_cast_to_geometry,
         are_dtypes_equal,
     )
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
@@ -1138,6 +1139,11 @@ def get_pipe_data(
         for col, typ in pipe.dtypes.items()
         if typ == 'bytes' and col in dtypes
     ]
+    geometry_columns = [
+        col
+        for col, typ in pipe.dtypes.items()
+        if typ.startswith('geometry') and col in dtypes
+    ]
 
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
 
@@ -1162,6 +1168,11 @@ def get_pipe_data(
             continue
         df[col] = df[col].apply(attempt_cast_to_bytes)
 
+    for col in geometry_columns:
+        if col not in df.columns:
+            continue
+        df[col] = df[col].apply(attempt_cast_to_geometry)
+
     if self.flavor == 'sqlite':
         ignore_dt_cols = [
             col
@@ -1511,7 +1522,7 @@ def get_pipe_attributes(
             if isinstance(parameters, str) and parameters[0] == '{':
                 parameters = json.loads(parameters)
             attributes['parameters'] = parameters
-        except Exception as e:
+        except Exception:
             attributes['parameters'] = {}
 
     return attributes

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,7 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -798,6 +798,7 @@ def to_sql(
         get_numeric_cols,
         get_uuid_cols,
         get_bytes_cols,
+        get_geometry_cols,
     )
     from meerschaum.utils.dtypes import (
         are_dtypes_equal,
@@ -805,7 +806,9 @@ def to_sql(
         encode_bytes_for_bytea,
         serialize_bytes,
         serialize_decimal,
+        serialize_geometry,
         json_serialize_value,
+        get_geometry_type_srid,
     )
     from meerschaum.utils.dtypes.sql import (
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
@@ -822,6 +825,7 @@ def to_sql(
 
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
+    geometry_cols = get_geometry_cols(df)
     ### NOTE: This excludes non-numeric serialized Decimals (e.g. SQLite).
     numeric_cols_dtypes = {
         col: typ
@@ -830,7 +834,6 @@ def to_sql(
             col in df.columns
             and 'numeric' in str(typ).lower()
         )
-        
     }
     numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
     numeric_cols_precisions_scales = {
@@ -841,6 +844,22 @@ def to_sql(
         )
         for col, typ in numeric_cols_dtypes.items()
     }
+    geometry_cols_dtypes = {
+        col: typ
+        for col, typ in kw.get('dtype', {}).items()
+        if (
+            col in df.columns
+            and 'geometry' in str(typ).lower() or 'geography' in str(typ).lower()
+        )
+    }
+    geometry_cols.extend([col for col in geometry_cols_dtypes if col not in geometry_cols])
+    geometry_cols_types_srids = {
+        col: (typ.geometry_type, typ.srid)
+        if hasattr(typ, 'srid')
+        else get_geometry_type_srid()
+        for col, typ in geometry_cols_dtypes.items()
+    }
+
     cols_pd_types = {
         col: get_pd_type_from_db_type(str(typ))
         for col, typ in kw.get('dtype', {}).items()
@@ -856,8 +875,9 @@ def to_sql(
     }
 
     enable_bulk_insert = mrsm.get_config(
-        'system', 'connectors', 'sql', 'bulk_insert'
-    ).get(self.flavor, False)
+        'system', 'connectors', 'sql', 'bulk_insert', self.flavor,
+        warn=False,
+    ) or False
     stats = {'target': name}
     ### resort to defaults if None
     copied = False
@@ -901,6 +921,17 @@ def to_sql(
             )
         )
 
+    for col in geometry_cols:
+        geometry_type, srid = geometry_cols_types_srids.get(col, get_geometry_type_srid())
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            df[col] = df[col].apply(
+                functools.partial(
+                    serialize_geometry,
+                    as_wkt=(self.flavor == 'mssql')
+                )
+            )
+
     stats['method'] = method.__name__ if hasattr(method, '__name__') else str(method)
 
     default_chunksize = self._sys_config.get('chunksize', None)

{meerschaum-2.9.0.dev1 → meerschaum-2.9.0rc1}/meerschaum/utils/dataframe.py

@@ -153,6 +153,7 @@ def filter_unseen_df(
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
+        attempt_cast_to_geometry,
         coerce_timezone,
         serialize_decimal,
     )
@@ -350,6 +351,10 @@ def filter_unseen_df(
     new_bytes_cols = get_bytes_cols(new_df)
     bytes_cols = set(new_bytes_cols + old_bytes_cols)
 
+    old_geometry_cols = get_geometry_cols(old_df)
+    new_geometry_cols = get_geometry_cols(new_df)
+    geometry_cols = set(new_geometry_cols + old_geometry_cols)
+
     joined_df = merge(
         new_df.infer_objects(copy=False).fillna(NA),
         old_df.infer_objects(copy=False).fillna(NA),
@@ -400,6 +405,14 @@ def filter_unseen_df(
         except Exception:
             warn(f"Unable to parse bytes column '{bytes_col}':\n{traceback.format_exc()}")
 
+    for geometry_col in geometry_cols:
+        if geometry_col not in delta_df.columns:
+            continue
+        try:
+            delta_df[geometry_col] = delta_df[geometry_col].apply(attempt_cast_to_geometry)
+        except Exception:
+            warn(f"Unable to parse bytes column '{bytes_col}':\n{traceback.format_exc()}")
+
     return delta_df
 
 
@@ -858,6 +871,44 @@ def get_bytes_cols(df: 'pd.DataFrame') -> List[str]:
     ]
 
 
+def get_geometry_cols(df: 'pd.DataFrame') -> List[str]:
+    """
+    Get the columns which contain shapely objects from a Pandas DataFrame.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame which may contain bytes strings.
+
+    Returns
+    -------
+    A list of columns to treat as `geometry`.
+    """
+    if df is None:
+        return []
+
+    is_dask = 'dask' in df.__module__
+    if is_dask:
+        df = get_first_valid_dask_partition(df)
+
+    if len(df) == 0:
+        return []
+
+    cols_indices = {
+        col: df[col].first_valid_index()
+        for col in df.columns
+    }
+    return [
+        col
+        for col, ix in cols_indices.items()
+        if (
+            ix is not None
+            and
+            'shapely' in str(type(df.loc[ix][col]))
+        )
+    ]
+
+
 def enforce_dtypes(
     df: 'pd.DataFrame',
     dtypes: Dict[str, str],
@@ -911,6 +962,7 @@ def enforce_dtypes(
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
+        attempt_cast_to_geometry,
         coerce_timezone as _coerce_timezone,
     )
     from meerschaum.utils.dtypes.sql import get_numeric_precision_scale
@@ -937,6 +989,11 @@ def enforce_dtypes(
         for col, typ in dtypes.items()
         if typ.startswith('numeric')
     ]
+    geometry_cols = [
+        col
+        for col, typ in dtypes.items()
+        if typ.startswith('geometry') or typ.startswith('geography')
+    ]
     uuid_cols = [
         col
         for col, typ in dtypes.items()
@@ -1026,6 +1083,24 @@ def enforce_dtypes(
             if col in df.columns:
                 df[col] = _coerce_timezone(df[col], strip_utc=strip_timezone)
 
+    if geometry_cols:
+        geopandas = mrsm.attempt_import('geopandas')
+        if debug:
+            dprint(f"Checking for geometry: {geometry_cols}")
+        parsed_geom_cols = []
+        for col in geometry_cols:
+            try:
+                df[col] = df[col].apply(attempt_cast_to_geometry)
+                parsed_geom_cols.append(col)
+            except Exception as e:
+                if debug:
+                    dprint(f"Unable to parse column '{col}' as geometry:\n{e}")
+
+        if parsed_geom_cols:
+            if debug:
+                dprint(f"Converting to GeoDataFrame (geometry column: '{parsed_geom_cols[0]}')...")
+            df = geopandas.GeoDataFrame(df, geometry=parsed_geom_cols[0])
+
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if are_dtypes_equal(df_dtypes, pipe_pandas_dtypes):
         if debug:
@@ -1602,13 +1677,19 @@ def to_json(
     -------
     A JSON string.
     """
+    import warnings
     from meerschaum.utils.packages import import_pandas
-    from meerschaum.utils.dtypes import serialize_bytes, serialize_decimal
+    from meerschaum.utils.dtypes import (
+        serialize_bytes,
+        serialize_decimal,
+        serialize_geometry,
+    )
     pd = import_pandas()
     uuid_cols = get_uuid_cols(df)
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
-    if safe_copy and bool(uuid_cols or bytes_cols):
+    geometry_cols = get_geometry_cols(df)
+    if safe_copy and bool(uuid_cols or bytes_cols or geometry_cols or numeric_cols):
         df = df.copy()
     for col in uuid_cols:
         df[col] = df[col].astype(str)
@@ -1616,6 +1697,10 @@ def to_json(
         df[col] = df[col].apply(serialize_bytes)
     for col in numeric_cols:
         df[col] = df[col].apply(serialize_decimal)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        for col in geometry_cols:
+            df[col] = df[col].apply(serialize_geometry)
     return df.infer_objects(copy=False).fillna(pd.NA).to_json(
         date_format=date_format,
         date_unit=date_unit,