meerschaum 3.0.8__py3-none-any.whl → 3.0.10__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -1564,7 +1564,7 @@ def create_pipe_table_from_df(
     )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     if self.flavor == 'geopackage':
-        init_success, init_msg = self._init_geopackage_table(df, pipe.target, debug=debug)
+        init_success, init_msg = self._init_geopackage_pipe(df, pipe, debug=debug)
         if not init_success:
             return init_success, init_msg
 
@@ -1625,7 +1625,7 @@ def create_pipe_table_from_df(
         else f"Failed to create {target_name}."
     )
     if success and self.flavor == 'geopackage':
-        return self._init_geopackage_table(df, target, debug=debug)
+        return self._init_geopackage_pipe(df, pipe, debug=debug)
 
     return success, msg
 
@@ -3092,6 +3092,7 @@ def get_pipe_columns_types(
         return {}
 
     if self.flavor not in ('oracle', 'mysql', 'mariadb', 'sqlite', 'geopackage'):
+    #  if self.flavor not in ('oracle', 'mysql', 'mariadb'):
         return get_table_cols_types(
             pipe.target,
             self,
@@ -3358,6 +3359,11 @@ def get_alter_columns_queries(
             'date': 'datetime',
             'numeric': 'int',
         })
+    elif self.flavor == 'geopackage':
+        pd_db_df_aliases.update({
+            'geometry': 'bytes',
+            'bytes': 'geometry',
+        })
 
     altered_cols = {
         col: (db_cols_types.get(col, 'object'), typ)
@@ -3939,3 +3945,234 @@ def _enforce_pipe_dtypes_chunks_hook(
     Enforce a pipe's dtypes on each chunk.
     """
     return pipe.enforce_dtypes(chunk_df, debug=debug)
+
+
+def _init_geopackage_pipe(
+    self,
+    df: 'pd.DataFrame',
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> SuccessTuple:
+    """
+    Initialize the geopackage schema tables from a DataFrame.
+    """
+    import pathlib
+    import shutil
+    from meerschaum.utils.sql import (
+        get_table_cols_types,
+        get_table_cols_indices,
+        get_create_table_queries,
+        table_exists,
+    )
+    from meerschaum.utils.dtypes import get_geometry_type_srid
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
+    from meerschaum.utils.misc import generate_password
+    from meerschaum.config.paths import SQL_CONN_CACHE_RESOURCES_PATH
+    from meerschaum.connectors import connectors
+    database = self.__dict__.get('database', self.parse_uri(self.URI).get('database', None))
+    if not database:
+        return False, f"Could not determine database for '{self}'."
+
+    table = pipe.target
+
+    tmp_id = generate_password(8)
+    self_sqlite = mrsm.get_connector(
+        f"sql:{self.label}_{tmp_id}",
+        flavor='sqlite',
+        database=database,
+    )
+    database_path = pathlib.Path(database)
+    create_new = False
+    failed = False
+    mode = 'w' if not database_path.exists() else 'a'
+    table_is_fresh = not table_exists(table, self_sqlite, debug=debug)
+    geom_cols_types = {
+        col: get_geometry_type_srid(typ)
+        for col, typ in pipe.dtypes.items()
+        if col and 'geometry' in col.lower() or 'geography' in col.lower()
+    }
+
+    try:
+        df.head(0).to_file(
+            database_path.as_posix(),
+            layer=table,
+            driver='GPKG',
+            index=False,
+            mode=mode,
+            spatial_index=False,
+        )
+    except Exception as e:
+        failed = True
+        create_new = 'at least' in str(e).lower()
+        if not create_new:
+            return False, f"Failed to init table '{table}':\n{e}"
+
+    rename_queries = [
+        (
+            "UPDATE gpkg_geometry_columns\n"
+            f"SET column_name = '{df.active_geometry_name}'\n"
+            f"WHERE table_name = '{table}'\n"
+            "  AND column_name = 'geom'"
+        ),
+    ]
+    for col, (geom_typ, srid) in geom_cols_types.items():
+        rename_queries.extend([
+            (
+                "UPDATE gpkg_geometry_columns\n"
+                f"SET geometry_type_name = '{geom_typ.upper()}'\n"
+                f"WHERE table_name = '{table}'\n"
+                f"  AND column_name = '{col}'"
+            ),
+        ])
+
+    if table_is_fresh:
+        rename_queries.append(f"DROP TABLE IF EXISTS \"{table}\"")
+
+    if not failed and not create_new:
+        self_sqlite.exec_queries(rename_queries, debug=debug)
+
+    if not create_new:
+        return (
+            True, "Success"   
+        ) if not failed else (
+            False, f"Failed to init {self}."
+        )
+
+    tmp_dir_path = SQL_CONN_CACHE_RESOURCES_PATH / tmp_id
+    tmp_dir_path.mkdir(parents=True, exist_ok=False)
+    temp_database_path = tmp_dir_path / f'{tmp_id}.gpkg'
+    tmp_conn = mrsm.get_connector(
+        f"sql:{tmp_id}",
+        flavor='sqlite',
+        database=temp_database_path.as_posix(),
+    )
+    df.head(0).to_file(
+        temp_database_path.as_posix(),
+        layer=table,
+        driver='GPKG',
+        mode='w',
+        spatial_index=False,
+    )
+    tmp_conn.exec_queries(rename_queries, debug=debug)
+    if not table_exists('geometry_columns', self_sqlite, debug=debug):
+        self.exec(
+            "CREATE VIEW geometry_columns AS\n"
+            "SELECT\n"
+            "    table_name AS f_table_name,\n"
+            "    column_name AS f_geometry_column,\n"
+            "    geometry_type_name AS \"type\",\n"
+            "    CASE\n"
+            "        WHEN z = 0 THEN 'XY'\n"
+            "        WHEN z = 1 THEN 'XYZ'\n"
+            "        WHEN z = 2 THEN 'XYZ'\n"
+            "    END AS coord_dimension,\n"
+            "    srs_id AS srid,\n"
+            "    0 AS spatial_index_enabled\n"
+            "FROM gpkg_geometry_columns"
+        )
+
+    tables_definitions = tmp_conn.read(
+        "SELECT name, sql\n"
+        "FROM sqlite_master\n"
+        "WHERE type = 'table'\n"
+        r"  AND name LIKE 'gpkg_%' "
+        f"AND name != '{table}'"
+    )
+    tables = tables_definitions['name']
+    tables_cols_types = {
+        tbl: get_table_cols_types(tbl, tmp_conn, flavor='sqlite', debug=debug)
+        for tbl in tables
+    }
+    tables_cols_indices = {
+        tbl: get_table_cols_indices(tbl, tmp_conn, flavor='sqlite', debug=debug)
+        for tbl in tables
+    }
+    tables_pks = {}
+    for tbl, cols_indices in tables_cols_indices.items():
+        for col, indices in cols_indices.items():
+            for ix_dict in indices:
+                if ix_dict.get('type') == 'PRIMARY KEY':
+                    tables_pks[tbl] = col
+                    break
+
+            if tbl in tables_pks:
+                break
+
+        if tbl in tables_pks:
+            break
+
+    tables_create_queries = {
+        tbl: get_create_table_queries(
+            tables_cols_types[tbl],
+            tbl,
+            flavor='sqlite',
+            primary_key=tables_pks.get(tbl, None),
+            primary_key_db_type=tables_cols_types[tbl].get(tables_pks.get(tbl, None)),
+            _parse_dtypes=False,
+        )
+        for tbl in tables
+    }
+    tables_pipes = {
+        tbl: mrsm.Pipe(
+            f'{tmp_conn}', tbl,
+            temporary=True,
+            instance=str(self_sqlite),
+            target=tbl,
+            static=True,
+            enforce=True,
+            null_indices=False,
+            upsert=False,
+            columns={
+                (col if tables_pks.get(tbl) != col else 'primary'): col
+                for col in tables_cols_indices[tbl]
+            },
+            dtypes={
+                col: (
+                    get_pd_type_from_db_type(
+                        db_typ
+                        if 'datetime' not in db_typ.lower()
+                        else 'TIMESTAMPTZ'
+                    )
+                )
+                for col, db_typ in tables_cols_types[tbl].items()
+            },
+            parameters={
+                'sql': f"SELECT * FROM \"{tbl}\"",
+            },
+        )
+        for tbl, cols_types in tables_cols_types.items()
+    }
+
+    init_success = True
+
+    for tbl, create_queries in tables_create_queries.items():
+        result = self_sqlite.exec_queries(create_queries, debug=debug)
+        if not result:
+            warn(f"Failed to create '{table}'.")
+            init_success = False
+
+        pipe = tables_pipes[tbl]
+        sync_success, sync_message = pipe.sync(debug=debug)
+        if not sync_success:
+            warn(f"Failed to migrate table '{tbl}':\n{sync_message}")
+            init_success = False
+
+        if not pipe.indices:
+            continue
+
+        index_success, index_message = pipe.create_indices(debug=debug)
+        if not index_success:
+            warn(f"Failed to create indices for '{tbl}':\n{index_message}")
+            init_success = False
+
+    failed = not init_success
+    _ = connectors.get('sql', {}).pop(tmp_id, None)
+    shutil.rmtree(tmp_dir_path)
+
+    _ = connectors.get('sql', {}).pop(self_sqlite.label, None)
+
+
+    if failed:
+        return False, f"Failed to init {table}."
+
+    return True, "Success"

meerschaum/connectors/sql/_sql.py

@@ -946,6 +946,11 @@ def to_sql(
             )
         )
 
+    geometry_format = 'wkt' if self.flavor == 'mssql' else (
+        'gpkg_wkb'
+        if self.flavor == 'geopackage'
+        else 'wkb_hex'
+    )
     for col in geometry_cols:
         geometry_type, srid = geometry_cols_types_srids.get(col, get_geometry_type_srid())
         with warnings.catch_warnings():
@@ -953,7 +958,7 @@ def to_sql(
             df[col] = df[col].apply(
                 functools.partial(
                     serialize_geometry,
-                    geometry_format=('wkt' if self.flavor == 'mssql' else 'wkb_hex'),
+                    geometry_format=geometry_format,
                 )
             )
 
@@ -1347,33 +1352,3 @@ def _cleanup_connections(self) -> None:
             connection.close()
         except Exception:
             pass
-
-
-def _init_geopackage_table(
-    self,
-    df: 'pd.DataFrame',
-    table: str,
-    debug: bool = False,
-) -> SuccessTuple:
-    """
-    Initialize the geopackage schema tables from a DataFrame.
-    """
-    import pathlib
-    database = self.__dict__.get('database', self.parse_uri(self.URI).get('database', None))
-    if not database:
-        return False, f"Could not determine database for '{self}'."
-
-    database_path = pathlib.Path(database)
-    mode = 'w' if not database_path.exists() else 'a'
-
-    try:
-        df.head(0).to_file(
-            database_path.as_posix(),
-            layer=table,
-            driver='GPKG',
-            index=False,
-            mode=mode,
-        )
-    except Exception as e:
-        return False, f"Failed to init table '{table}':\n{e}"
-    return True, "Success"

meerschaum/utils/dtypes/__init__.py

@@ -10,6 +10,7 @@ import traceback
 import json
 import uuid
 import time
+import struct
 from datetime import timezone, datetime, date, timedelta
 from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
 
@@ -360,6 +361,14 @@ def attempt_cast_to_geometry(value: Any) -> Any:
         except Exception:
             return value
 
+    value_is_gpkg = geometry_is_gpkg(value)
+    if value_is_gpkg:
+        try:
+            wkb_data, _, _ = gpkg_wkb_to_wkb(value)
+            return shapely_wkb.loads(wkb_data)
+        except Exception:
+            return value
+
     value_is_wkt = geometry_is_wkt(value)
     if value_is_wkt is None:
         return value
@@ -405,6 +414,56 @@ def geometry_is_wkt(value: Union[str, bytes]) -> Union[bool, None]:
     return None
 
 
+def geometry_is_gpkg(value: bytes) -> bool:
+    """
+    Return whether the input `value` is formatted as GeoPackage WKB.
+    """
+    if not isinstance(value, bytes) or len(value) < 2:
+        return False
+
+    return value[0:2] == b'GP'
+
+def gpkg_wkb_to_wkb(gpkg_wkb_bytes: bytes) -> Tuple[bytes, int, bytes]:
+    """
+    Converts GeoPackage WKB to standard WKB by removing the header.
+    
+    Parameters
+    ----------
+    gpkg_wkb_bytes: bytes
+        The GeoPackage WKB byte string.
+        
+    Returns
+    -------
+    A tuple containing the standard WKB bytes, SRID, and flags.
+    """
+    magic_number = gpkg_wkb_bytes[0:2]
+    if magic_number != b'GP':
+        raise ValueError("Invalid GeoPackage WKB header: missing magic number.")
+    
+    try:
+        header = gpkg_wkb_bytes[0:8]
+        header_vals = struct.unpack('<ccBBi', header)
+        flags = header_vals[-2]
+        srid = header_vals[-1]
+    except struct.error:
+        header = gpkg_wkb_bytes[0:6]
+        header_vals = struct.unpack('<ccBBh', header)
+        flags = header_vals[-2]
+        srid = header_vals[-1]
+
+    envelope_type = (flags >> 1) & 0x07
+    envelope_sizes = {
+        0: 0,
+        1: 32,
+        2: 48,
+        3: 48,
+        4: 64,
+    }
+    header_length = 8 + envelope_sizes.get(envelope_type, 0)
+    standard_wkb_bytes = gpkg_wkb_bytes[header_length:]
+    return standard_wkb_bytes, srid, flags
+
+
 def value_is_null(value: Any) -> bool:
     """
     Determine if a value is a null-like string.
@@ -690,9 +749,9 @@ def serialize_geometry(
     geom: Any,
     geometry_format: str = 'wkb_hex',
     srid: Optional[int] = None,
-) -> Union[str, Dict[str, Any], None]:
+) -> Union[str, Dict[str, Any], bytes, None]:
     """
-    Serialize geometry data as a hex-encoded well-known-binary string. 
+    Serialize geometry data as WKB, WKB (hex), GPKG-WKB, WKT, or GeoJSON. 
 
     Parameters
     ----------
@@ -701,20 +760,20 @@ def serialize_geometry(
 
     geometry_format: str, default 'wkb_hex'
         The serialization format for geometry data.
-        Accepted formats are `wkb_hex` (well-known binary hex string),
-        `wkt` (well-known text), and `geojson`.
+        Accepted formats are `wkb`, `wkb_hex`, `wkt`, `geojson`, and `gpkg_wkb`.
 
     srid: Optional[int], default None
         If provided, use this as the source CRS when serializing to GeoJSON.
 
     Returns
     -------
-    A string containing the geometry data.
+    A string containing the geometry data, or bytes, or a dictionary, or None.
     """
     if value_is_null(geom):
         return None
-    shapely, shapely_ops, pyproj = mrsm.attempt_import(
-        'shapely', 'shapely.ops', 'pyproj',
+
+    shapely, shapely_ops, pyproj, np = mrsm.attempt_import(
+        'shapely', 'shapely.ops', 'pyproj', 'numpy',
         lazy=False,
     )
     if geometry_format == 'geojson':
@@ -724,12 +783,31 @@ def serialize_geometry(
         geojson_str = shapely.to_geojson(geom)
         return json.loads(geojson_str)
 
-    if hasattr(geom, 'wkb_hex'):
-        if geometry_format == "wkb_hex":
-            return shapely.to_wkb(geom, hex=True, include_srid=True)
-        return shapely.to_wkt(geom)
+    if not hasattr(geom, 'wkb_hex'):
+        return str(geom)
+
+    byte_order = 1 if np.little_endian else 0
+
+    if geometry_format.startswith("wkb"):
+        return shapely.to_wkb(geom, hex=(geometry_format=="wkb_hex"), include_srid=True)
+    elif geometry_format == 'gpkg_wkb':
+        wkb_data = shapely.to_wkb(geom, hex=False, byte_order=byte_order)
+        flags = (
+            ((byte_order & 0x01) | (0x20))
+            if geom.is_empty
+            else (byte_order & 0x01)
+        )
+        srid_val = srid or -1
+        header = struct.pack(
+            '<ccBBi',
+            b'G', b'P',
+            0,
+            flags,
+            srid_val
+        )
+        return header + wkb_data
 
-    return str(geom)
+    return shapely.to_wkt(geom)
 
 
 def deserialize_geometry(geom_wkb: Union[str, bytes]):

meerschaum/utils/dtypes/sql.py

@@ -635,7 +635,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mssql': 'NVARCHAR(MAX)',
         'oracle': 'NVARCHAR2(2000)',
         'sqlite': 'TEXT',
-        'geopackage': 'GEOMETRY',
+        'geopackage': 'BLOB',
         'duckdb': 'TEXT',
         'citus': 'TEXT',
         'cockroachdb': 'TEXT',
@@ -651,7 +651,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mssql': 'NVARCHAR(MAX)',
         'oracle': 'NVARCHAR2(2000)',
         'sqlite': 'TEXT',
-        'geopackage': 'TEXT',
+        'geopackage': 'BLOB',
         'duckdb': 'TEXT',
         'citus': 'TEXT',
         'cockroachdb': 'TEXT',
@@ -1133,7 +1133,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mssql': 'UnicodeText',
         'oracle': 'UnicodeText',
         'sqlite': 'UnicodeText',
-        'geopackage': 'UnicodeText',
+        'geopackage': 'LargeBinary',
         'duckdb': 'UnicodeText',
         'citus': 'UnicodeText',
         'cockroachdb': 'UnicodeText',

meerschaum/utils/sql.py

@@ -2146,6 +2146,7 @@ def get_create_table_queries(
     primary_key_db_type: Optional[str] = None,
     autoincrement: bool = False,
     datetime_column: Optional[str] = None,
+    _parse_dtypes: bool = True,
 ) -> List[str]:
     """
     Return a query to create a new table from a `SELECT` query or a `dtypes` dictionary.
@@ -2179,6 +2180,10 @@ def get_create_table_queries(
         If provided, include this column in the primary key.
         Applicable to TimescaleDB only.
 
+    _parse_dtypes: bool, default True
+        If `True`, cast Pandas dtypes to SQL dtypes.
+        Otherwise pass through the given value directly.
+
     Returns
     -------
     A `CREATE TABLE` (or `SELECT INTO`) query for the database flavor.
@@ -2200,6 +2205,7 @@ def get_create_table_queries(
         primary_key_db_type=primary_key_db_type,
         autoincrement=(autoincrement and flavor not in SKIP_AUTO_INCREMENT_FLAVORS),
         datetime_column=datetime_column,
+        _parse_dtypes=_parse_dtypes,
     )
 
 
@@ -2212,6 +2218,7 @@ def _get_create_table_query_from_dtypes(
     primary_key_db_type: Optional[str] = None,
     autoincrement: bool = False,
     datetime_column: Optional[str] = None,
+    _parse_dtypes: bool = True,
 ) -> List[str]:
     """
     Create a new table from a `dtypes` dictionary.
@@ -2228,12 +2235,22 @@ def _get_create_table_query_from_dtypes(
             (
                 primary_key,
                 get_db_type_from_pd_type(dtypes.get(primary_key, 'int') or 'int', flavor=flavor)
+            ) if _parse_dtypes else (
+                primary_key,
+                dtypes.get(primary_key, 'INT') or 'INT'
             )
         ]
         if primary_key
         else []
     ) + [
-        (col, get_db_type_from_pd_type(typ, flavor=flavor))
+        (
+            col,
+            (
+                get_db_type_from_pd_type(typ, flavor=flavor)
+                if _parse_dtypes
+                else typ
+            )
+        )
         for col, typ in dtypes.items()
         if col != primary_key
     ]
@@ -2311,6 +2328,7 @@ def _get_create_table_query_from_cte(
     primary_key_db_type: Optional[str] = None,
     autoincrement: bool = False,
     datetime_column: Optional[str] = None,
+    _parse_dtypes=None,
 ) -> List[str]:
     """
     Create a new table from a CTE query.

{meerschaum-3.0.8.dist-info → meerschaum-3.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meerschaum
-Version: 3.0.8
+Version: 3.0.10
 Summary: Sync Time-Series Pipes with Meerschaum
 Author-email: Bennett Meares <bennett.meares@gmail.com>
 Maintainer-email: Bennett Meares <bennett.meares@gmail.com>