meerschaum 2.9.0rc2__py3-none-any.whl → 2.9.1__py3-none-any.whl

meerschaum/plugins/__init__.py

@@ -91,8 +91,8 @@ def make_action(
 
 
 def pre_sync_hook(
-        function: Callable[[Any], Any],
-    ) -> Callable[[Any], Any]:
+    function: Callable[[Any], Any],
+) -> Callable[[Any], Any]:
     """
     Register a function as a sync hook to be executed right before sync.
     
@@ -169,6 +169,7 @@ def web_page(
     page: Union[str, None, Callable[[Any], Any]] = None,
     login_required: bool = True,
     skip_navbar: bool = False,
+    page_group: Optional[str] = None,
     **kwargs
 ) -> Any:
     """
@@ -188,7 +189,7 @@ def web_page(
     page_str = None
 
     def _decorator(_func: Callable[[Any], Any]) -> Callable[[Any], Any]:
-        nonlocal page_str
+        nonlocal page_str, page_group
 
         @functools.wraps(_func)
         def wrapper(*_args, **_kwargs):
@@ -198,10 +199,31 @@ def web_page(
             page_str = _func.__name__
 
         page_str = page_str.lstrip('/').rstrip('/').strip()
-        _plugin_endpoints_to_pages[page_str] = {
+        page_key = (
+            ' '.join(
+                [
+                    word.capitalize()
+                    for word in (
+                        page_str.replace('/dash', '').lstrip('/').rstrip('/').strip()
+                        .replace('-', ' ').replace('_', ' ').split(' ')
+                    )
+                ]
+            )
+        )
+ 
+        package_name = _func.__globals__['__name__']
+        plugin_name = (
+            package_name.split('.')[1]
+            if package_name.startswith('plugins.') else None
+        )
+        page_group = page_group or plugin_name
+        if page_group not in _plugin_endpoints_to_pages:
+            _plugin_endpoints_to_pages[page_group] = {}
+        _plugin_endpoints_to_pages[page_group][page_str] = {
             'function': _func,
             'login_required': login_required,
             'skip_navbar': skip_navbar,
+            'page_key': page_key,
         }
         return wrapper
 

meerschaum/utils/dataframe.py

@@ -871,7 +871,10 @@ def get_bytes_cols(df: 'pd.DataFrame') -> List[str]:
     ]
 
 
-def get_geometry_cols(df: 'pd.DataFrame') -> List[str]:
+def get_geometry_cols(
+    df: 'pd.DataFrame',
+    with_types_srids: bool = False,
+) -> Union[List[str], Dict[str, Any]]:
     """
     Get the columns which contain shapely objects from a Pandas DataFrame.
 
@@ -880,9 +883,13 @@ def get_geometry_cols(df: 'pd.DataFrame') -> List[str]:
     df: pd.DataFrame
         The DataFrame which may contain bytes strings.
 
+    with_types_srids: bool, default False
+        If `True`, return a dictionary mapping columns to geometry types and SRIDs.
+
     Returns
     -------
     A list of columns to treat as `geometry`.
+    If `with_types_srids`, return a dictionary mapping columns to tuples in the form (type, SRID).
     """
     if df is None:
         return []
@@ -898,7 +905,7 @@ def get_geometry_cols(df: 'pd.DataFrame') -> List[str]:
         col: df[col].first_valid_index()
         for col in df.columns
     }
-    return [
+    geo_cols = [
         col
         for col, ix in cols_indices.items()
         if (
@@ -907,6 +914,38 @@ def get_geometry_cols(df: 'pd.DataFrame') -> List[str]:
             'shapely' in str(type(df.loc[ix][col]))
         )
     ]
+    if not with_types_srids:
+        return geo_cols
+
+    gpd = mrsm.attempt_import('geopandas', lazy=False)
+    geo_cols_types_srids = {}
+    for col in geo_cols:
+        try:
+            sample_geo_series = gpd.GeoSeries(df[col], crs=None)
+            geometry_types = {
+                geom.geom_type
+                for geom in sample_geo_series
+                if hasattr(geom, 'geom_type')
+            }
+            geometry_has_z = any(getattr(geom, 'has_z', False) for geom in sample_geo_series)
+            srid = (
+                (
+                    sample_geo_series.crs.sub_crs_list[0].to_epsg()
+                    if sample_geo_series.crs.is_compound
+                    else sample_geo_series.crs.to_epsg()
+                )
+                if sample_geo_series.crs
+                else 0
+            )
+            geometry_type = list(geometry_types)[0] if len(geometry_types) == 1 else 'geometry'
+            if geometry_type != 'geometry' and geometry_has_z:
+                geometry_type = geometry_type + 'Z'
+        except Exception:
+            srid = 0
+            geometry_type = 'geometry'
+        geo_cols_types_srids[col] = (geometry_type, srid)
+
+    return geo_cols_types_srids
 
 
 def enforce_dtypes(
@@ -1658,6 +1697,8 @@ def to_json(
     orient: str = 'records',
     date_format: str = 'iso',
     date_unit: str = 'us',
+    double_precision: int = 15,
+    geometry_format: str = 'geojson',
     **kwargs: Any
 ) -> str:
     """
@@ -1677,11 +1718,19 @@ def to_json(
     date_unit: str, default 'us'
         The precision of the timestamps.
 
+    double_precision: int, default 15
+        The number of decimal places to use when encoding floating point values (maximum 15).
+
+    geometry_format: str, default 'geojson'
+        The serialization format for geometry data.
+        Accepted values are `geojson`, `wkb_hex`, and `wkt`.
+
     Returns
     -------
     A JSON string.
     """
     import warnings
+    import functools
     from meerschaum.utils.packages import import_pandas
     from meerschaum.utils.dtypes import (
         serialize_bytes,
@@ -1704,10 +1753,16 @@ def to_json(
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         for col in geometry_cols:
-            df[col] = df[col].apply(serialize_geometry)
+            df[col] = df[col].apply(
+                functools.partial(
+                    serialize_geometry,
+                    geometry_format=geometry_format,
+                )
+            )
     return df.infer_objects(copy=False).fillna(pd.NA).to_json(
         date_format=date_format,
         date_unit=date_unit,
+        double_precision=double_precision,
         orient=orient,
         **kwargs
     )

meerschaum/utils/dtypes/__init__.py

@@ -7,6 +7,7 @@ Utility functions for working with data types.
 """
 
 import traceback
+import json
 import uuid
 from datetime import timezone, datetime
 from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
@@ -28,6 +29,7 @@ MRSM_ALIAS_DTYPES: Dict[str, str] = {
     'guid': 'uuid',
     'UUID': 'uuid',
     'geom': 'geometry',
+    'geog': 'geography',
 }
 MRSM_PD_DTYPES: Dict[Union[str, None], str] = {
     'json': 'object',
@@ -76,6 +78,7 @@ def to_pandas_dtype(dtype: str) -> str:
         return get_pd_type_from_db_type(dtype)
 
     from meerschaum.utils.packages import attempt_import
+    _ = attempt_import('pyarrow', lazy=False)
     pandas = attempt_import('pandas', lazy=False)
 
     try:
@@ -294,10 +297,21 @@ def attempt_cast_to_geometry(value: Any) -> Any:
     """
     Given a value, attempt to coerce it into a `shapely` (`geometry`) object.
     """
-    shapely_wkt, shapely_wkb = mrsm.attempt_import('shapely.wkt', 'shapely.wkb', lazy=False)
+    shapely, shapely_wkt, shapely_wkb = mrsm.attempt_import(
+        'shapely',
+        'shapely.wkt',
+        'shapely.wkb',
+        lazy=False,
+    )
     if 'shapely' in str(type(value)):
         return value
 
+    if isinstance(value, (dict, list)):
+        try:
+            return shapely.from_geojson(json.dumps(value))
+        except Exception as e:
+            return value
+
     value_is_wkt = geometry_is_wkt(value)
     if value_is_wkt is None:
         return value
@@ -327,6 +341,9 @@ def geometry_is_wkt(value: Union[str, bytes]) -> Union[bool, None]:
     Return `None` if `value` should be parsed as neither.
     """
     import re
+    if not isinstance(value, (str, bytes)):
+        return None
+
     if isinstance(value, bytes):
         return False
     
@@ -521,7 +538,11 @@ def serialize_bytes(data: bytes) -> str:
     return base64.b64encode(data).decode('utf-8')
 
 
-def serialize_geometry(geom: Any, as_wkt: bool = False) -> str:
+def serialize_geometry(
+    geom: Any,
+    geometry_format: str = 'wkb_hex',
+    as_wkt: bool = False,
+) -> Union[str, Dict[str, Any]]:
     """
     Serialize geometry data as a hex-encoded well-known-binary string. 
 
@@ -530,16 +551,22 @@ def serialize_geometry(geom: Any, as_wkt: bool = False) -> str:
     geom: Any
         The potential geometry data to be serialized.
 
-    as_wkt, bool, default False
-        If `True`, serialize geometry data as well-known text (WKT)
-        instead of well-known binary (WKB).
+    geometry_format: str, default 'wkb_hex'
+        The serialization format for geometry data.
+        Accepted formats are `wkb_hex` (well-known binary hex string),
+        `wkt` (well-known text), and `geojson`.
 
     Returns
     -------
     A string containing the geometry data.
     """
+    shapely = mrsm.attempt_import('shapely', lazy=False)
+    if geometry_format == 'geojson':
+        geojson_str = shapely.to_geojson(geom)
+        return json.loads(geojson_str)
+
     if hasattr(geom, 'wkb_hex'):
-        return geom.wkb_hex if not as_wkt else geom.wkt
+        return geom.wkb_hex if geometry_format == 'wkb_hex' else geom.wkt
 
     return str(geom)
 
@@ -711,24 +738,25 @@ def get_geometry_type_srid(
     ('Point', 4376)
     """
     from meerschaum.utils.misc import is_int
+    ### NOTE: PostGIS syntax must also be parsed.
+    dtype = dtype.replace('(', '[').replace(')', ']')
     bare_dtype = dtype.split('[', maxsplit=1)[0]
     modifier = dtype.split(bare_dtype, maxsplit=1)[-1].lstrip('[').rstrip(']')
     if not modifier:
         return default_type, default_srid
 
-    shapely_geometry_base = mrsm.attempt_import('shapely.geometry.base')
-    geometry_types = {
-        typ.lower(): typ
-        for typ in shapely_geometry_base.GEOMETRY_TYPES
-    }
-
-    parts = [part.lower().replace('srid=', '').replace('type=', '').strip() for part in modifier.split(',')]
+    parts = [
+        part.split('=')[-1].strip()
+        for part in modifier.split(',')
+    ]
     parts_casted = [
         (
             int(part)
             if is_int(part)
             else part
-        ) for part in parts]
+        )
+        for part in parts
+    ]
 
     srid = default_srid
     geometry_type = default_type
@@ -738,9 +766,9 @@ def get_geometry_type_srid(
             srid = part
             break
 
-    for part in parts:
-        if part.lower() in geometry_types:
-            geometry_type = geometry_types.get(part)
+    for part in parts_casted:
+        if isinstance(part, str):
+            geometry_type = part
             break
 
     return geometry_type, srid

meerschaum/utils/dtypes/sql.py

@@ -74,10 +74,12 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
     'DOUBLE': 'float64[pyarrow]',
     'DECIMAL': 'numeric',
     'BIGINT': 'int64[pyarrow]',
-    'INT': 'int64[pyarrow]',
-    'INTEGER': 'int64[pyarrow]',
+    'INT': 'int32[pyarrow]',
+    'INTEGER': 'int32[pyarrow]',
     'NUMBER': 'numeric',
     'NUMERIC': 'numeric',
+    'GEOMETRY': 'geometry',
+    'GEOMETRY(GEOMETRY)': 'geometry',
     'TIMESTAMP': 'datetime64[ns]',
     'TIMESTAMP WITHOUT TIMEZONE': 'datetime64[ns]',
     'TIMESTAMP WITH TIMEZONE': 'datetime64[ns, UTC]',
@@ -120,6 +122,8 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
         'BYTE': 'bytes',
         'LOB': 'bytes',
         'BINARY': 'bytes',
+        'GEOMETRY': 'geometry',
+        'GEOGRAPHY': 'geography',
     },
     'default': 'object',
 }
@@ -139,6 +143,90 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'cockroachdb': 'BIGINT',
         'default': 'INT',
     },
+    'uint': {
+        'timescaledb': 'BIGINT',
+        'postgresql': 'BIGINT',
+        'postgis': 'BIGINT',
+        'mariadb': 'BIGINT',
+        'mysql': 'BIGINT',
+        'mssql': 'BIGINT',
+        'oracle': 'INT',
+        'sqlite': 'BIGINT',
+        'duckdb': 'BIGINT',
+        'citus': 'BIGINT',
+        'cockroachdb': 'BIGINT',
+        'default': 'INT',
+    },
+    'int8': {
+        'timescaledb': 'SMALLINT',
+        'postgresql': 'SMALLINT',
+        'postgis': 'SMALLINT',
+        'mariadb': 'SMALLINT',
+        'mysql': 'SMALLINT',
+        'mssql': 'SMALLINT',
+        'oracle': 'INT',
+        'sqlite': 'INT',
+        'duckdb': 'SMALLINT',
+        'citus': 'SMALLINT',
+        'cockroachdb': 'SMALLINT',
+        'default': 'INT',
+    },
+    'uint8': {
+        'timescaledb': 'SMALLINT',
+        'postgresql': 'SMALLINT',
+        'postgis': 'SMALLINT',
+        'mariadb': 'SMALLINT',
+        'mysql': 'SMALLINT',
+        'mssql': 'SMALLINT',
+        'oracle': 'INT',
+        'sqlite': 'INT',
+        'duckdb': 'SMALLINT',
+        'citus': 'SMALLINT',
+        'cockroachdb': 'SMALLINT',
+        'default': 'INT',
+    },
+    'int16': {
+        'timescaledb': 'SMALLINT',
+        'postgresql': 'SMALLINT',
+        'postgis': 'SMALLINT',
+        'mariadb': 'SMALLINT',
+        'mysql': 'SMALLINT',
+        'mssql': 'SMALLINT',
+        'oracle': 'INT',
+        'sqlite': 'INT',
+        'duckdb': 'SMALLINT',
+        'citus': 'SMALLINT',
+        'cockroachdb': 'SMALLINT',
+        'default': 'INT',
+    },
+    'int32': {
+        'timescaledb': 'INT',
+        'postgresql': 'INT',
+        'postgis': 'INT',
+        'mariadb': 'INT',
+        'mysql': 'INT',
+        'mssql': 'INT',
+        'oracle': 'INT',
+        'sqlite': 'INT',
+        'duckdb': 'INT',
+        'citus': 'INT',
+        'cockroachdb': 'INT',
+        'default': 'INT',
+    },
+    'int64': {
+        'timescaledb': 'BIGINT',
+        'postgresql': 'BIGINT',
+        'postgis': 'BIGINT',
+        'mariadb': 'BIGINT',
+        'mysql': 'BIGINT',
+        'mssql': 'BIGINT',
+        'oracle': 'INT',
+        'sqlite': 'BIGINT',
+        'duckdb': 'BIGINT',
+        'citus': 'BIGINT',
+        'cockroachdb': 'BIGINT',
+        'default': 'INT',
+    },
     'float': {
         'timescaledb': 'DOUBLE PRECISION',
         'postgresql': 'DOUBLE PRECISION',
@@ -380,6 +468,90 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'cockroachdb': 'BigInteger',
         'default': 'BigInteger',
     },
+    'uint': {
+        'timescaledb': 'BigInteger',
+        'postgresql': 'BigInteger',
+        'postgis': 'BigInteger',
+        'mariadb': 'BigInteger',
+        'mysql': 'BigInteger',
+        'mssql': 'BigInteger',
+        'oracle': 'BigInteger',
+        'sqlite': 'BigInteger',
+        'duckdb': 'BigInteger',
+        'citus': 'BigInteger',
+        'cockroachdb': 'BigInteger',
+        'default': 'BigInteger',
+    },
+    'int8': {
+        'timescaledb': 'SmallInteger',
+        'postgresql': 'SmallInteger',
+        'postgis': 'SmallInteger',
+        'mariadb': 'SmallInteger',
+        'mysql': 'SmallInteger',
+        'mssql': 'SmallInteger',
+        'oracle': 'SmallInteger',
+        'sqlite': 'SmallInteger',
+        'duckdb': 'SmallInteger',
+        'citus': 'SmallInteger',
+        'cockroachdb': 'SmallInteger',
+        'default': 'SmallInteger',
+    },
+    'uint8': {
+        'timescaledb': 'SmallInteger',
+        'postgresql': 'SmallInteger',
+        'postgis': 'SmallInteger',
+        'mariadb': 'SmallInteger',
+        'mysql': 'SmallInteger',
+        'mssql': 'SmallInteger',
+        'oracle': 'SmallInteger',
+        'sqlite': 'SmallInteger',
+        'duckdb': 'SmallInteger',
+        'citus': 'SmallInteger',
+        'cockroachdb': 'SmallInteger',
+        'default': 'SmallInteger',
+    },
+    'int16': {
+        'timescaledb': 'SmallInteger',
+        'postgresql': 'SmallInteger',
+        'postgis': 'SmallInteger',
+        'mariadb': 'SmallInteger',
+        'mysql': 'SmallInteger',
+        'mssql': 'SmallInteger',
+        'oracle': 'SmallInteger',
+        'sqlite': 'SmallInteger',
+        'duckdb': 'SmallInteger',
+        'citus': 'SmallInteger',
+        'cockroachdb': 'SmallInteger',
+        'default': 'SmallInteger',
+    },
+    'int32': {
+        'timescaledb': 'Integer',
+        'postgresql': 'Integer',
+        'postgis': 'Integer',
+        'mariadb': 'Integer',
+        'mysql': 'Integer',
+        'mssql': 'Integer',
+        'oracle': 'Integer',
+        'sqlite': 'Integer',
+        'duckdb': 'Integer',
+        'citus': 'Integer',
+        'cockroachdb': 'Integer',
+        'default': 'Integer',
+    },
+    'int64': {
+        'timescaledb': 'BigInteger',
+        'postgresql': 'BigInteger',
+        'postgis': 'BigInteger',
+        'mariadb': 'BigInteger',
+        'mysql': 'BigInteger',
+        'mssql': 'BigInteger',
+        'oracle': 'BigInteger',
+        'sqlite': 'BigInteger',
+        'duckdb': 'BigInteger',
+        'citus': 'BigInteger',
+        'cockroachdb': 'BigInteger',
+        'default': 'BigInteger',
+    },
     'float': {
         'timescaledb': 'Float',
         'postgresql': 'Float',
@@ -596,7 +768,7 @@ def get_pd_type_from_db_type(db_type: str, allow_custom_dtypes: bool = True) ->
     -------
     The equivalent datatype for a pandas DataFrame.
     """
-    from meerschaum.utils.dtypes import are_dtypes_equal
+    from meerschaum.utils.dtypes import are_dtypes_equal, get_geometry_type_srid
     def parse_custom(_pd_type: str, _db_type: str) -> str:
         if 'json' in _db_type.lower():
             return 'json'
@@ -604,6 +776,13 @@ def get_pd_type_from_db_type(db_type: str, allow_custom_dtypes: bool = True) ->
             precision, scale = get_numeric_precision_scale(None, dtype=_db_type.upper())
             if precision and scale:
                 return f"numeric[{precision},{scale}]"
+        if are_dtypes_equal(_pd_type, 'geometry') and _pd_type != 'object':
+            geometry_type, srid = get_geometry_type_srid(_db_type.upper())
+            modifiers = [str(modifier) for modifier in (geometry_type, srid) if modifier]
+            typ = "geometry" if 'geography' not in _pd_type.lower() else 'geography'
+            if not modifiers:
+                return typ
+            return f"{typ}[{', '.join(modifiers)}]"
         return _pd_type
 
     pd_type = DB_TO_PD_DTYPES.get(db_type.upper().split('(', maxsplit=1)[0].strip(), None)

meerschaum/utils/misc.py

@@ -175,7 +175,7 @@ def string_to_dict(
     import ast
     params_dict = {}
     for param in params_string.split(","):
-        _keys = param.split(":")
+        _keys = param.split(":", maxsplit=1)
         keys = _keys[:-1]
         try:
             val = ast.literal_eval(_keys[-1])

meerschaum/utils/packages/_packages.py

@@ -88,6 +88,11 @@ packages: Dict[str, Dict[str, str]] = {
         'mssqlcli'                   : 'mssql-cli>=1.0.0',
         'gadwall'                    : 'gadwall>=0.2.0',
     },
+    'gis'                            : {
+        'pyproj'                     : 'pyproj>=3.7.1',
+        'geopandas'                  : 'geopandas>=1.0.1', 
+        'shapely'                    : 'shapely>=2.0.7',
+    },
     'stack': {
         'compose'                    : 'docker-compose>=1.29.2',
     },
@@ -120,7 +125,6 @@ packages: Dict[str, Dict[str, str]] = {
         'jinja2'                     : 'jinja2==3.0.3',
     },
     'gui': {
-        'toga'                       : 'toga>=0.3.0-dev29',
         'webview'                    : 'pywebview>=3.6.3',
         'pycparser'                  : 'pycparser>=2.21.0',
     },
@@ -135,8 +139,6 @@ packages: Dict[str, Dict[str, str]] = {
 packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'geopandas'                      : 'geopandas>=1.0.1', 
-    'shapely'                        : 'shapely>=2.0.7',
     'pyarrow'                        : 'pyarrow>=16.1.0',
     'dask'                           : 'dask[complete]>=2024.12.1',
     'partd'                          : 'partd>=1.4.2',
@@ -150,6 +152,7 @@ packages['sql'] = {
 }
 packages['sql'].update(packages['drivers'])
 packages['sql'].update(packages['core'])
+packages['sql'].update(packages['gis'])
 packages['dash'] = {
     'flask_compress'                 : 'Flask-Compress>=1.10.1',
     'dash'                           : 'dash>=2.6.2',