meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/utils/dataframe.py

@@ -1003,6 +1003,7 @@ def enforce_dtypes(
         attempt_cast_to_bytes,
         attempt_cast_to_geometry,
         coerce_timezone as _coerce_timezone,
+        get_geometry_type_srid,
     )
     from meerschaum.utils.dtypes.sql import get_numeric_precision_scale
     pandas = mrsm.attempt_import('pandas')
@@ -1028,11 +1029,11 @@ def enforce_dtypes(
         for col, typ in dtypes.items()
         if typ.startswith('numeric')
     ]
-    geometry_cols = [
-        col
+    geometry_cols_types_srids = {
+        col: get_geometry_type_srid(typ, default_srid=0)
         for col, typ in dtypes.items()
         if typ.startswith('geometry') or typ.startswith('geography')
-    ]
+    }
     uuid_cols = [
         col
         for col, typ in dtypes.items()
@@ -1120,14 +1121,24 @@ def enforce_dtypes(
             dprint(f"Checking for datetime conversion: {datetime_cols}")
         for col in datetime_cols:
             if col in df.columns:
+                if debug:
+                    dprint(
+                        f"Data type for column '{col}' before timezone coersion: "
+                        f"{str(df[col].dtype)}"
+                    )
                 df[col] = _coerce_timezone(df[col], strip_utc=strip_timezone)
+                if debug:
+                    dprint(
+                        f"Data type for column '{col}' after timezone coersion: "
+                        f"{str(df[col].dtype)}"
+                    )
 
-    if geometry_cols:
+    if geometry_cols_types_srids:
         geopandas = mrsm.attempt_import('geopandas')
         if debug:
-            dprint(f"Checking for geometry: {geometry_cols}")
+            dprint(f"Checking for geometry: {list(geometry_cols_types_srids)}")
         parsed_geom_cols = []
-        for col in geometry_cols:
+        for col in geometry_cols_types_srids:
             try:
                 df[col] = df[col].apply(attempt_cast_to_geometry)
                 parsed_geom_cols.append(col)
@@ -1139,10 +1150,19 @@ def enforce_dtypes(
             if debug:
                 dprint(f"Converting to GeoDataFrame (geometry column: '{parsed_geom_cols[0]}')...")
             try:
-                df = geopandas.GeoDataFrame(df, geometry=parsed_geom_cols[0])
-                df.rename_geometry(parsed_geom_cols[0], inplace=True)
+                _, default_srid = geometry_cols_types_srids[parsed_geom_cols[0]]
+                df = geopandas.GeoDataFrame(df, geometry=parsed_geom_cols[0], crs=default_srid)
+                for col, (_, srid) in geometry_cols_types_srids.items():
+                    if srid:
+                        if debug:
+                            dprint(f"Setting '{col}' to SRID '{srid}'...")
+                        _ = df[col].set_crs(srid)
+                if parsed_geom_cols[0] not in df.columns:
+                    df.rename_geometry(parsed_geom_cols[0], inplace=True)
             except (ValueError, TypeError):
-                pass
+                if debug:
+                    import traceback
+                    dprint(f"Failed to cast to GeoDataFrame:\n{traceback.format_exc()}")
 
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if are_dtypes_equal(df_dtypes, pipe_pandas_dtypes):
@@ -1386,7 +1406,7 @@ def chunksize_to_npartitions(chunksize: Optional[int]) -> int:
 
 def df_from_literal(
     pipe: Optional[mrsm.Pipe] = None,
-    literal: str = None,
+    literal: Optional[str] = None,
     debug: bool = False
 ) -> 'pd.DataFrame':
     """
@@ -1408,8 +1428,8 @@ def df_from_literal(
 
     if pipe is None or literal is None:
         error("Please provide a Pipe and a literal value")
-    ### this will raise an error if the columns are undefined
-    dt_name, val_name = pipe.get_columns('datetime', 'value')
+    dt_col = pipe.columns.get('datetime', 'ts')
+    val_col = pipe.get_val_column(debug=debug)
 
     val = literal
     if isinstance(literal, str):
@@ -1429,7 +1449,7 @@ def df_from_literal(
     now = datetime.now(timezone.utc).replace(tzinfo=None)
 
     pd = import_pandas()
-    return pd.DataFrame({dt_name: [now], val_name: [val]})
+    return pd.DataFrame({dt_col: [now], val_col: [val]})
 
 
 def get_first_valid_dask_partition(ddf: 'dask.dataframe.DataFrame') -> Union['pd.DataFrame', None]:
@@ -1533,9 +1553,14 @@ def query_df(
     NA = pandas.NA
 
     if params:
+        proto_in_ex_params = get_in_ex_params(params)
+        for key, (proto_in_vals, proto_ex_vals) in proto_in_ex_params.items():
+            if proto_ex_vals:
+                coerce_types = True
+                break
         params = params.copy()
         for key, val in {k: v for k, v in params.items()}.items():
-            if isinstance(val, (list, tuple)):
+            if isinstance(val, (list, tuple, set)) or hasattr(val, 'astype'):
                 if None in val:
                     val = [item for item in val if item is not None] + [NA]
                     params[key] = val
@@ -1742,6 +1767,10 @@ def to_json(
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
     geometry_cols = get_geometry_cols(df)
+    geometry_cols_srids = {
+        col: int((getattr(df[col].crs, 'srs', '') or '').split(':', maxsplit=1)[-1] or '0')
+        for col in geometry_cols
+    } if 'geodataframe' in str(type(df)).lower() else {}
     if safe_copy and bool(uuid_cols or bytes_cols or geometry_cols or numeric_cols):
         df = df.copy()
     for col in uuid_cols:
@@ -1753,10 +1782,12 @@ def to_json(
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         for col in geometry_cols:
+            srid = geometry_cols_srids.get(col, None) or None
             df[col] = df[col].apply(
                 functools.partial(
                     serialize_geometry,
                     geometry_format=geometry_format,
+                    srid=srid,
                 )
             )
     return df.infer_objects(copy=False).fillna(pd.NA).to_json(
@@ -1766,3 +1797,53 @@ def to_json(
         orient=orient,
         **kwargs
     )
+
+
+def to_simple_lines(df: 'pd.DataFrame') -> str:
+    """
+    Serialize a Pandas Dataframe as lines of simple dictionaries.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The dataframe to serialize into simple lines text.
+
+    Returns
+    -------
+    A string of simple line dictionaries joined by newlines.
+    """
+    from meerschaum.utils.misc import to_simple_dict
+    if df is None or len(df) == 0:
+        return ''
+
+    docs = df.to_dict(orient='records')
+    return '\n'.join(to_simple_dict(doc) for doc in docs)
+
+
+def parse_simple_lines(data: str) -> 'pd.DataFrame':
+    """
+    Parse simple lines text into a DataFrame.
+
+    Parameters
+    ----------
+    data: str
+        The simple lines text to parse into a DataFrame.
+
+    Returns
+    -------
+    A dataframe containing the rows serialized in `data`.
+    """
+    from meerschaum.utils.misc import string_to_dict
+    from meerschaum.utils.packages import import_pandas
+    pd = import_pandas()
+    lines = data.splitlines()
+    try:
+        docs = [string_to_dict(line) for line in lines]
+        df = pd.DataFrame(docs)
+    except Exception:
+        df = None
+
+    if df is None:
+        raise ValueError("Cannot parse simple lines into a dataframe.")
+
+    return df

meerschaum/utils/dtypes/__init__.py

@@ -309,7 +309,7 @@ def attempt_cast_to_geometry(value: Any) -> Any:
     if isinstance(value, (dict, list)):
         try:
             return shapely.from_geojson(json.dumps(value))
-        except Exception as e:
+        except Exception:
             return value
 
     value_is_wkt = geometry_is_wkt(value)
@@ -361,7 +361,7 @@ def value_is_null(value: Any) -> bool:
     """
     Determine if a value is a null-like string.
     """
-    return str(value).lower() in ('none', 'nan', 'na', 'nat', '', '<na>')
+    return str(value).lower() in ('none', 'nan', 'na', 'nat', 'natz', '', '<na>')
 
 
 def none_if_null(value: Any) -> Any:
@@ -455,7 +455,10 @@ def coerce_timezone(
 
     if isinstance(dt, str):
         dateutil_parser = mrsm.attempt_import('dateutil.parser')
-        dt = dateutil_parser.parse(dt)
+        try:
+            dt = dateutil_parser.parse(dt)
+        except Exception:
+            return dt
 
     dt_is_series = hasattr(dt, 'dtype') and hasattr(dt, '__module__')
 
@@ -472,6 +475,8 @@ def coerce_timezone(
             return dt
 
         dt_series = to_datetime(dt, coerce_utc=False)
+        if dt_series.dt.tz is None:
+            dt_series = dt_series.dt.tz_localize(timezone.utc)
         if strip_utc:
             try:
                 if dt_series.dt.tz is not None:
@@ -502,25 +507,69 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
     dt_is_series = hasattr(dt_val, 'dtype') and hasattr(dt_val, '__module__')
     pd = pandas if dd is None else dd
 
-    try:
-        new_dt_val = pd.to_datetime(dt_val, utc=True, format='ISO8601')
-        if as_pydatetime:
-            return new_dt_val.to_pydatetime()
-        return new_dt_val
-    except (pd.errors.OutOfBoundsDatetime, ValueError):
-        pass
-
     def parse(x: Any) -> Any:
         try:
             return dateutil_parser.parse(x)
         except Exception:
             return x
 
+    if isinstance(dt_val, pd.Timestamp):
+        dt_val_to_return = dt_val if not as_pydatetime else dt_val.to_pydatetime()
+        return (
+            coerce_timezone(dt_val_to_return)
+            if coerce_utc
+            else dt_val_to_return
+        )
+
     if dt_is_series:
-        new_series = dt_val.apply(parse)
+        changed_tz = False
+        original_tz = None
+        dtype = str(getattr(dt_val, 'dtype', 'object'))
+        if are_dtypes_equal(dtype, 'datetime') and 'utc' not in dtype.lower():
+            original_tz = dt_val.dt.tz
+            dt_val = dt_val.dt.tz_localize(timezone.utc)
+            changed_tz = True
+            dtype = str(getattr(dt_val, 'dtype', 'object'))
+        try:
+            new_dt_series = (
+                dt_val
+                if dtype == 'datetime64[ns, UTC]'
+                else dt_val.astype("datetime64[ns, UTC]")
+            )
+        except pd.errors.OutOfBoundsDatetime:
+            try:
+                new_dt_series = dt_val.astype("datetime64[ms, UTC]")
+            except Exception:
+                new_dt_series = None
+        except ValueError:
+            new_dt_series = None
+        except TypeError:
+            try:
+                new_dt_series = (
+                    new_dt_series
+                    if str(getattr(new_dt_series, 'dtype', None)) == 'datetime64[ns]'
+                    else dt_val.astype("datetime64[ns]")
+                )
+            except Exception:
+                new_dt_series = None
+
+        if new_dt_series is None:
+            new_dt_series = dt_val.apply(lambda x: parse(str(x)))
+
         if coerce_utc:
-            return coerce_timezone(new_series)
-        return new_series
+            return coerce_timezone(new_dt_series)
+
+        if changed_tz:
+            new_dt_series = new_dt_series.dt.tz_localize(original_tz)
+        return new_dt_series
+
+    try:
+        new_dt_val = pd.to_datetime(dt_val, utc=True, format='ISO8601')
+        if as_pydatetime:
+            return new_dt_val.to_pydatetime()
+        return new_dt_val
+    except (pd.errors.OutOfBoundsDatetime, ValueError):
+        pass
 
     new_dt_val = parse(dt_val)
     if not coerce_utc:
@@ -541,6 +590,7 @@ def serialize_bytes(data: bytes) -> str:
 def serialize_geometry(
     geom: Any,
     geometry_format: str = 'wkb_hex',
+    srid: Optional[int] = None,
 ) -> Union[str, Dict[str, Any], None]:
     """
     Serialize geometry data as a hex-encoded well-known-binary string. 
@@ -555,19 +605,30 @@ def serialize_geometry(
         Accepted formats are `wkb_hex` (well-known binary hex string),
         `wkt` (well-known text), and `geojson`.
 
+    srid: Optional[int], default None
+        If provided, use this as the source CRS when serializing to GeoJSON.
+
     Returns
     -------
     A string containing the geometry data.
     """
     if value_is_null(geom):
         return None
-    shapely = mrsm.attempt_import('shapely', lazy=False)
+    shapely, shapely_ops, pyproj = mrsm.attempt_import(
+        'shapely', 'shapely.ops', 'pyproj',
+        lazy=False,
+    )
     if geometry_format == 'geojson':
+        if srid:
+            transformer = pyproj.Transformer.from_crs(f"EPSG:{srid}", "EPSG:4326", always_xy=True)
+            geom = shapely_ops.transform(transformer.transform, geom)
         geojson_str = shapely.to_geojson(geom)
         return json.loads(geojson_str)
 
     if hasattr(geom, 'wkb_hex'):
-        return geom.wkb_hex if geometry_format == 'wkb_hex' else geom.wkt
+        if geometry_format == "wkb_hex":
+            return shapely.to_wkb(geom, hex=True, include_srid=True)
+        return shapely.to_wkt(geom)
 
     return str(geom)
 
@@ -576,10 +637,19 @@ def deserialize_geometry(geom_wkb: Union[str, bytes]):
     """
     Deserialize a WKB string into a shapely geometry object.
     """
-    shapely = mrsm.attempt_import(lazy=False)
+    shapely = mrsm.attempt_import('shapely', lazy=False)
     return shapely.wkb.loads(geom_wkb)
 
 
+def project_geometry(geom, srid: int, to_srid: int = 4326):
+    """
+    Project a shapely geometry object to a new CRS (SRID).
+    """
+    pyproj, shapely_ops = mrsm.attempt_import('pyproj', 'shapely.ops', lazy=False)
+    transformer = pyproj.Transformer.from_crs(f"EPSG:{srid}", f"EPSG:{to_srid}", always_xy=True)
+    return shapely_ops.transform(transformer.transform, geom)
+
+
 def deserialize_bytes_string(data: Optional[str], force_hex: bool = False) -> Union[bytes, None]:
     """
     Given a serialized ASCII string of bytes data, return the original bytes.
@@ -652,7 +722,7 @@ def serialize_datetime(dt: datetime) -> Union[str, None]:
     return dt.isoformat() + tz_suffix
 
 
-def json_serialize_value(x: Any, default_to_str: bool = True) -> str:
+def json_serialize_value(x: Any, default_to_str: bool = True) -> Union[str, None]:
     """
     Serialize the given value to a JSON value. Accounts for datetimes, bytes, decimals, etc.
 
@@ -687,6 +757,9 @@ def json_serialize_value(x: Any, default_to_str: bool = True) -> str:
     if value_is_null(x):
         return None
 
+    if isinstance(x, (dict, list, tuple)):
+        return json.dumps(x, default=json_serialize_value, separators=(',', ':'))
+
     return str(x) if default_to_str else x