meerschaum 3.0.8__py3-none-any.whl → 3.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/bootstrap.py +10 -1
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/sql/_SQLConnector.py +1 -1
- meerschaum/connectors/sql/_pipes.py +239 -2
- meerschaum/connectors/sql/_sql.py +6 -31
- meerschaum/utils/dtypes/__init__.py +90 -12
- meerschaum/utils/dtypes/sql.py +3 -3
- meerschaum/utils/sql.py +19 -1
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/METADATA +1 -1
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/RECORD +17 -17
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/WHEEL +0 -0
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/entry_points.txt +0 -0
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/licenses/NOTICE +0 -0
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/top_level.txt +0 -0
- {meerschaum-3.0.8.dist-info → meerschaum-3.0.9.dist-info}/zip-safe +0 -0
meerschaum/actions/bootstrap.py
CHANGED
@@ -422,8 +422,17 @@ def _bootstrap_plugins(
|
|
422
422
|
"""
|
423
423
|
Launch an interactive wizard to guide the user to creating a new plugin.
|
424
424
|
"""
|
425
|
-
from meerschaum.utils.prompt import prompt
|
425
|
+
from meerschaum.utils.prompt import prompt, yes_no
|
426
426
|
from meerschaum.plugins.bootstrap import bootstrap_plugin
|
427
|
+
from meerschaum.config.paths import PLUGINS_DIR_PATHS
|
428
|
+
|
429
|
+
for dir_path in PLUGINS_DIR_PATHS:
|
430
|
+
if not dir_path.exists():
|
431
|
+
create_dir_path = yes_no(f"Create path '{dir_path}'?", default='y')
|
432
|
+
if create_dir_path:
|
433
|
+
dir_path.mkdir(parents=True, exist_ok=True)
|
434
|
+
else:
|
435
|
+
return False, "Create missing plugins paths before bootstrapping."
|
427
436
|
|
428
437
|
if not action:
|
429
438
|
action = [prompt("Enter the name of your new plugin:")]
|
meerschaum/config/_default.py
CHANGED
meerschaum/config/_version.py
CHANGED
@@ -36,7 +36,6 @@ class SQLConnector(InstanceConnector):
|
|
36
36
|
exec_queries,
|
37
37
|
get_connection,
|
38
38
|
_cleanup_connections,
|
39
|
-
_init_geopackage_table,
|
40
39
|
)
|
41
40
|
from meerschaum.utils.sql import test_connection
|
42
41
|
from ._fetch import fetch, get_pipe_metadef
|
@@ -74,6 +73,7 @@ class SQLConnector(InstanceConnector):
|
|
74
73
|
create_pipe_indices,
|
75
74
|
drop_pipe_indices,
|
76
75
|
get_pipe_index_names,
|
76
|
+
_init_geopackage_pipe,
|
77
77
|
)
|
78
78
|
from ._plugins import (
|
79
79
|
get_plugins_pipe,
|
@@ -1564,7 +1564,7 @@ def create_pipe_table_from_df(
|
|
1564
1564
|
)
|
1565
1565
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
1566
1566
|
if self.flavor == 'geopackage':
|
1567
|
-
init_success, init_msg = self.
|
1567
|
+
init_success, init_msg = self._init_geopackage_pipe(df, pipe, debug=debug)
|
1568
1568
|
if not init_success:
|
1569
1569
|
return init_success, init_msg
|
1570
1570
|
|
@@ -1625,7 +1625,7 @@ def create_pipe_table_from_df(
|
|
1625
1625
|
else f"Failed to create {target_name}."
|
1626
1626
|
)
|
1627
1627
|
if success and self.flavor == 'geopackage':
|
1628
|
-
return self.
|
1628
|
+
return self._init_geopackage_pipe(df, pipe, debug=debug)
|
1629
1629
|
|
1630
1630
|
return success, msg
|
1631
1631
|
|
@@ -3092,6 +3092,7 @@ def get_pipe_columns_types(
|
|
3092
3092
|
return {}
|
3093
3093
|
|
3094
3094
|
if self.flavor not in ('oracle', 'mysql', 'mariadb', 'sqlite', 'geopackage'):
|
3095
|
+
# if self.flavor not in ('oracle', 'mysql', 'mariadb'):
|
3095
3096
|
return get_table_cols_types(
|
3096
3097
|
pipe.target,
|
3097
3098
|
self,
|
@@ -3358,6 +3359,11 @@ def get_alter_columns_queries(
|
|
3358
3359
|
'date': 'datetime',
|
3359
3360
|
'numeric': 'int',
|
3360
3361
|
})
|
3362
|
+
elif self.flavor == 'geopackage':
|
3363
|
+
pd_db_df_aliases.update({
|
3364
|
+
'geometry': 'bytes',
|
3365
|
+
'bytes': 'geometry',
|
3366
|
+
})
|
3361
3367
|
|
3362
3368
|
altered_cols = {
|
3363
3369
|
col: (db_cols_types.get(col, 'object'), typ)
|
@@ -3939,3 +3945,234 @@ def _enforce_pipe_dtypes_chunks_hook(
|
|
3939
3945
|
Enforce a pipe's dtypes on each chunk.
|
3940
3946
|
"""
|
3941
3947
|
return pipe.enforce_dtypes(chunk_df, debug=debug)
|
3948
|
+
|
3949
|
+
|
3950
|
+
def _init_geopackage_pipe(
|
3951
|
+
self,
|
3952
|
+
df: 'pd.DataFrame',
|
3953
|
+
pipe: mrsm.Pipe,
|
3954
|
+
debug: bool = False,
|
3955
|
+
) -> SuccessTuple:
|
3956
|
+
"""
|
3957
|
+
Initialize the geopackage schema tables from a DataFrame.
|
3958
|
+
"""
|
3959
|
+
import pathlib
|
3960
|
+
import shutil
|
3961
|
+
from meerschaum.utils.sql import (
|
3962
|
+
get_table_cols_types,
|
3963
|
+
get_table_cols_indices,
|
3964
|
+
get_create_table_queries,
|
3965
|
+
table_exists,
|
3966
|
+
)
|
3967
|
+
from meerschaum.utils.dtypes import get_geometry_type_srid
|
3968
|
+
from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
|
3969
|
+
from meerschaum.utils.misc import generate_password
|
3970
|
+
from meerschaum.config.paths import SQL_CONN_CACHE_RESOURCES_PATH
|
3971
|
+
from meerschaum.connectors import connectors
|
3972
|
+
database = self.__dict__.get('database', self.parse_uri(self.URI).get('database', None))
|
3973
|
+
if not database:
|
3974
|
+
return False, f"Could not determine database for '{self}'."
|
3975
|
+
|
3976
|
+
table = pipe.target
|
3977
|
+
|
3978
|
+
tmp_id = generate_password(8)
|
3979
|
+
self_sqlite = mrsm.get_connector(
|
3980
|
+
f"sql:{self.label}_{tmp_id}",
|
3981
|
+
flavor='sqlite',
|
3982
|
+
database=database,
|
3983
|
+
)
|
3984
|
+
database_path = pathlib.Path(database)
|
3985
|
+
create_new = False
|
3986
|
+
failed = False
|
3987
|
+
mode = 'w' if not database_path.exists() else 'a'
|
3988
|
+
table_is_fresh = not table_exists(table, self_sqlite, debug=debug)
|
3989
|
+
geom_cols_types = {
|
3990
|
+
col: get_geometry_type_srid(typ)
|
3991
|
+
for col, typ in pipe.dtypes.items()
|
3992
|
+
if col and 'geometry' in col.lower() or 'geography' in col.lower()
|
3993
|
+
}
|
3994
|
+
|
3995
|
+
try:
|
3996
|
+
df.head(0).to_file(
|
3997
|
+
database_path.as_posix(),
|
3998
|
+
layer=table,
|
3999
|
+
driver='GPKG',
|
4000
|
+
index=False,
|
4001
|
+
mode=mode,
|
4002
|
+
spatial_index=False,
|
4003
|
+
)
|
4004
|
+
except Exception as e:
|
4005
|
+
failed = True
|
4006
|
+
create_new = 'at least' in str(e).lower()
|
4007
|
+
if not create_new:
|
4008
|
+
return False, f"Failed to init table '{table}':\n{e}"
|
4009
|
+
|
4010
|
+
rename_queries = [
|
4011
|
+
(
|
4012
|
+
"UPDATE gpkg_geometry_columns\n"
|
4013
|
+
f"SET column_name = '{df.active_geometry_name}'\n"
|
4014
|
+
f"WHERE table_name = '{table}'\n"
|
4015
|
+
" AND column_name = 'geom'"
|
4016
|
+
),
|
4017
|
+
]
|
4018
|
+
for col, (geom_typ, srid) in geom_cols_types.items():
|
4019
|
+
rename_queries.extend([
|
4020
|
+
(
|
4021
|
+
"UPDATE gpkg_geometry_columns\n"
|
4022
|
+
f"SET geometry_type_name = '{geom_typ.upper()}'\n"
|
4023
|
+
f"WHERE table_name = '{table}'\n"
|
4024
|
+
f" AND column_name = '{col}'"
|
4025
|
+
),
|
4026
|
+
])
|
4027
|
+
|
4028
|
+
if table_is_fresh:
|
4029
|
+
rename_queries.append(f"DROP TABLE IF EXISTS \"{table}\"")
|
4030
|
+
|
4031
|
+
if not failed and not create_new:
|
4032
|
+
self_sqlite.exec_queries(rename_queries, debug=debug)
|
4033
|
+
|
4034
|
+
if not create_new:
|
4035
|
+
return (
|
4036
|
+
True, "Success"
|
4037
|
+
) if not failed else (
|
4038
|
+
False, f"Failed to init {self}."
|
4039
|
+
)
|
4040
|
+
|
4041
|
+
tmp_dir_path = SQL_CONN_CACHE_RESOURCES_PATH / tmp_id
|
4042
|
+
tmp_dir_path.mkdir(parents=True, exist_ok=False)
|
4043
|
+
temp_database_path = tmp_dir_path / f'{tmp_id}.gpkg'
|
4044
|
+
tmp_conn = mrsm.get_connector(
|
4045
|
+
f"sql:{tmp_id}",
|
4046
|
+
flavor='sqlite',
|
4047
|
+
database=temp_database_path.as_posix(),
|
4048
|
+
)
|
4049
|
+
df.head(0).to_file(
|
4050
|
+
temp_database_path.as_posix(),
|
4051
|
+
layer=table,
|
4052
|
+
driver='GPKG',
|
4053
|
+
mode='w',
|
4054
|
+
spatial_index=False,
|
4055
|
+
)
|
4056
|
+
tmp_conn.exec_queries(rename_queries, debug=debug)
|
4057
|
+
if not table_exists('geometry_columns', self_sqlite, debug=debug):
|
4058
|
+
self.exec(
|
4059
|
+
"CREATE VIEW geometry_columns AS\n"
|
4060
|
+
"SELECT\n"
|
4061
|
+
" table_name AS f_table_name,\n"
|
4062
|
+
" column_name AS f_geometry_column,\n"
|
4063
|
+
" geometry_type_name AS \"type\",\n"
|
4064
|
+
" CASE\n"
|
4065
|
+
" WHEN z = 0 THEN 'XY'\n"
|
4066
|
+
" WHEN z = 1 THEN 'XYZ'\n"
|
4067
|
+
" WHEN z = 2 THEN 'XYZ'\n"
|
4068
|
+
" END AS coord_dimension,\n"
|
4069
|
+
" srs_id AS srid,\n"
|
4070
|
+
" 0 AS spatial_index_enabled\n"
|
4071
|
+
"FROM gpkg_geometry_columns"
|
4072
|
+
)
|
4073
|
+
|
4074
|
+
tables_definitions = tmp_conn.read(
|
4075
|
+
"SELECT name, sql\n"
|
4076
|
+
"FROM sqlite_master\n"
|
4077
|
+
"WHERE type = 'table'\n"
|
4078
|
+
r" AND name LIKE 'gpkg_%' "
|
4079
|
+
f"AND name != '{table}'"
|
4080
|
+
)
|
4081
|
+
tables = tables_definitions['name']
|
4082
|
+
tables_cols_types = {
|
4083
|
+
tbl: get_table_cols_types(tbl, tmp_conn, flavor='sqlite', debug=debug)
|
4084
|
+
for tbl in tables
|
4085
|
+
}
|
4086
|
+
tables_cols_indices = {
|
4087
|
+
tbl: get_table_cols_indices(tbl, tmp_conn, flavor='sqlite', debug=debug)
|
4088
|
+
for tbl in tables
|
4089
|
+
}
|
4090
|
+
tables_pks = {}
|
4091
|
+
for tbl, cols_indices in tables_cols_indices.items():
|
4092
|
+
for col, indices in cols_indices.items():
|
4093
|
+
for ix_dict in indices:
|
4094
|
+
if ix_dict.get('type') == 'PRIMARY KEY':
|
4095
|
+
tables_pks[tbl] = col
|
4096
|
+
break
|
4097
|
+
|
4098
|
+
if tbl in tables_pks:
|
4099
|
+
break
|
4100
|
+
|
4101
|
+
if tbl in tables_pks:
|
4102
|
+
break
|
4103
|
+
|
4104
|
+
tables_create_queries = {
|
4105
|
+
tbl: get_create_table_queries(
|
4106
|
+
tables_cols_types[tbl],
|
4107
|
+
tbl,
|
4108
|
+
flavor='sqlite',
|
4109
|
+
primary_key=tables_pks.get(tbl, None),
|
4110
|
+
primary_key_db_type=tables_cols_types[tbl].get(tables_pks.get(tbl, None)),
|
4111
|
+
_parse_dtypes=False,
|
4112
|
+
)
|
4113
|
+
for tbl in tables
|
4114
|
+
}
|
4115
|
+
tables_pipes = {
|
4116
|
+
tbl: mrsm.Pipe(
|
4117
|
+
f'{tmp_conn}', tbl,
|
4118
|
+
temporary=True,
|
4119
|
+
instance=str(self_sqlite),
|
4120
|
+
target=tbl,
|
4121
|
+
static=True,
|
4122
|
+
enforce=True,
|
4123
|
+
null_indices=False,
|
4124
|
+
upsert=False,
|
4125
|
+
columns={
|
4126
|
+
(col if tables_pks.get(tbl) != col else 'primary'): col
|
4127
|
+
for col in tables_cols_indices[tbl]
|
4128
|
+
},
|
4129
|
+
dtypes={
|
4130
|
+
col: (
|
4131
|
+
get_pd_type_from_db_type(
|
4132
|
+
db_typ
|
4133
|
+
if 'datetime' not in db_typ.lower()
|
4134
|
+
else 'TIMESTAMPTZ'
|
4135
|
+
)
|
4136
|
+
)
|
4137
|
+
for col, db_typ in tables_cols_types[tbl].items()
|
4138
|
+
},
|
4139
|
+
parameters={
|
4140
|
+
'sql': f"SELECT * FROM \"{tbl}\"",
|
4141
|
+
},
|
4142
|
+
)
|
4143
|
+
for tbl, cols_types in tables_cols_types.items()
|
4144
|
+
}
|
4145
|
+
|
4146
|
+
init_success = True
|
4147
|
+
|
4148
|
+
for tbl, create_queries in tables_create_queries.items():
|
4149
|
+
result = self_sqlite.exec_queries(create_queries, debug=debug)
|
4150
|
+
if not result:
|
4151
|
+
warn(f"Failed to create '{table}'.")
|
4152
|
+
init_success = False
|
4153
|
+
|
4154
|
+
pipe = tables_pipes[tbl]
|
4155
|
+
sync_success, sync_message = pipe.sync(debug=debug)
|
4156
|
+
if not sync_success:
|
4157
|
+
warn(f"Failed to migrate table '{tbl}':\n{sync_message}")
|
4158
|
+
init_success = False
|
4159
|
+
|
4160
|
+
if not pipe.indices:
|
4161
|
+
continue
|
4162
|
+
|
4163
|
+
index_success, index_message = pipe.create_indices(debug=debug)
|
4164
|
+
if not index_success:
|
4165
|
+
warn(f"Failed to create indices for '{tbl}':\n{index_message}")
|
4166
|
+
init_success = False
|
4167
|
+
|
4168
|
+
failed = not init_success
|
4169
|
+
_ = connectors.get('sql', {}).pop(tmp_id, None)
|
4170
|
+
shutil.rmtree(tmp_dir_path)
|
4171
|
+
|
4172
|
+
_ = connectors.get('sql', {}).pop(self_sqlite.label, None)
|
4173
|
+
|
4174
|
+
|
4175
|
+
if failed:
|
4176
|
+
return False, f"Failed to init {table}."
|
4177
|
+
|
4178
|
+
return True, "Success"
|
@@ -946,6 +946,11 @@ def to_sql(
|
|
946
946
|
)
|
947
947
|
)
|
948
948
|
|
949
|
+
geometry_format = 'wkt' if self.flavor == 'mssql' else (
|
950
|
+
'gpkg_wkb'
|
951
|
+
if self.flavor == 'geopackage'
|
952
|
+
else 'wkb_hex'
|
953
|
+
)
|
949
954
|
for col in geometry_cols:
|
950
955
|
geometry_type, srid = geometry_cols_types_srids.get(col, get_geometry_type_srid())
|
951
956
|
with warnings.catch_warnings():
|
@@ -953,7 +958,7 @@ def to_sql(
|
|
953
958
|
df[col] = df[col].apply(
|
954
959
|
functools.partial(
|
955
960
|
serialize_geometry,
|
956
|
-
geometry_format=
|
961
|
+
geometry_format=geometry_format,
|
957
962
|
)
|
958
963
|
)
|
959
964
|
|
@@ -1347,33 +1352,3 @@ def _cleanup_connections(self) -> None:
|
|
1347
1352
|
connection.close()
|
1348
1353
|
except Exception:
|
1349
1354
|
pass
|
1350
|
-
|
1351
|
-
|
1352
|
-
def _init_geopackage_table(
|
1353
|
-
self,
|
1354
|
-
df: 'pd.DataFrame',
|
1355
|
-
table: str,
|
1356
|
-
debug: bool = False,
|
1357
|
-
) -> SuccessTuple:
|
1358
|
-
"""
|
1359
|
-
Initialize the geopackage schema tables from a DataFrame.
|
1360
|
-
"""
|
1361
|
-
import pathlib
|
1362
|
-
database = self.__dict__.get('database', self.parse_uri(self.URI).get('database', None))
|
1363
|
-
if not database:
|
1364
|
-
return False, f"Could not determine database for '{self}'."
|
1365
|
-
|
1366
|
-
database_path = pathlib.Path(database)
|
1367
|
-
mode = 'w' if not database_path.exists() else 'a'
|
1368
|
-
|
1369
|
-
try:
|
1370
|
-
df.head(0).to_file(
|
1371
|
-
database_path.as_posix(),
|
1372
|
-
layer=table,
|
1373
|
-
driver='GPKG',
|
1374
|
-
index=False,
|
1375
|
-
mode=mode,
|
1376
|
-
)
|
1377
|
-
except Exception as e:
|
1378
|
-
return False, f"Failed to init table '{table}':\n{e}"
|
1379
|
-
return True, "Success"
|
@@ -10,6 +10,7 @@ import traceback
|
|
10
10
|
import json
|
11
11
|
import uuid
|
12
12
|
import time
|
13
|
+
import struct
|
13
14
|
from datetime import timezone, datetime, date, timedelta
|
14
15
|
from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
|
15
16
|
|
@@ -360,6 +361,14 @@ def attempt_cast_to_geometry(value: Any) -> Any:
|
|
360
361
|
except Exception:
|
361
362
|
return value
|
362
363
|
|
364
|
+
value_is_gpkg = geometry_is_gpkg(value)
|
365
|
+
if value_is_gpkg:
|
366
|
+
try:
|
367
|
+
wkb_data, _, _ = gpkg_wkb_to_wkb(value)
|
368
|
+
return shapely_wkb.loads(wkb_data)
|
369
|
+
except Exception:
|
370
|
+
return value
|
371
|
+
|
363
372
|
value_is_wkt = geometry_is_wkt(value)
|
364
373
|
if value_is_wkt is None:
|
365
374
|
return value
|
@@ -405,6 +414,56 @@ def geometry_is_wkt(value: Union[str, bytes]) -> Union[bool, None]:
|
|
405
414
|
return None
|
406
415
|
|
407
416
|
|
417
|
+
def geometry_is_gpkg(value: bytes) -> bool:
|
418
|
+
"""
|
419
|
+
Return whether the input `value` is formatted as GeoPackage WKB.
|
420
|
+
"""
|
421
|
+
if not isinstance(value, bytes) or len(value) < 2:
|
422
|
+
return False
|
423
|
+
|
424
|
+
return value[0:2] == b'GP'
|
425
|
+
|
426
|
+
def gpkg_wkb_to_wkb(gpkg_wkb_bytes: bytes) -> Tuple[bytes, int, bytes]:
|
427
|
+
"""
|
428
|
+
Converts GeoPackage WKB to standard WKB by removing the header.
|
429
|
+
|
430
|
+
Parameters
|
431
|
+
----------
|
432
|
+
gpkg_wkb_bytes: bytes
|
433
|
+
The GeoPackage WKB byte string.
|
434
|
+
|
435
|
+
Returns
|
436
|
+
-------
|
437
|
+
A tuple containing the standard WKB bytes, SRID, and flags.
|
438
|
+
"""
|
439
|
+
magic_number = gpkg_wkb_bytes[0:2]
|
440
|
+
if magic_number != b'GP':
|
441
|
+
raise ValueError("Invalid GeoPackage WKB header: missing magic number.")
|
442
|
+
|
443
|
+
try:
|
444
|
+
header = gpkg_wkb_bytes[0:8]
|
445
|
+
header_vals = struct.unpack('<ccBBi', header)
|
446
|
+
flags = header_vals[-2]
|
447
|
+
srid = header_vals[-1]
|
448
|
+
except struct.error:
|
449
|
+
header = gpkg_wkb_bytes[0:6]
|
450
|
+
header_vals = struct.unpack('<ccBBh', header)
|
451
|
+
flags = header_vals[-2]
|
452
|
+
srid = header_vals[-1]
|
453
|
+
|
454
|
+
envelope_type = (flags >> 1) & 0x07
|
455
|
+
envelope_sizes = {
|
456
|
+
0: 0,
|
457
|
+
1: 32,
|
458
|
+
2: 48,
|
459
|
+
3: 48,
|
460
|
+
4: 64,
|
461
|
+
}
|
462
|
+
header_length = 8 + envelope_sizes.get(envelope_type, 0)
|
463
|
+
standard_wkb_bytes = gpkg_wkb_bytes[header_length:]
|
464
|
+
return standard_wkb_bytes, srid, flags
|
465
|
+
|
466
|
+
|
408
467
|
def value_is_null(value: Any) -> bool:
|
409
468
|
"""
|
410
469
|
Determine if a value is a null-like string.
|
@@ -690,9 +749,9 @@ def serialize_geometry(
|
|
690
749
|
geom: Any,
|
691
750
|
geometry_format: str = 'wkb_hex',
|
692
751
|
srid: Optional[int] = None,
|
693
|
-
) -> Union[str, Dict[str, Any], None]:
|
752
|
+
) -> Union[str, Dict[str, Any], bytes, None]:
|
694
753
|
"""
|
695
|
-
Serialize geometry data as
|
754
|
+
Serialize geometry data as WKB, WKB (hex), GPKG-WKB, WKT, or GeoJSON.
|
696
755
|
|
697
756
|
Parameters
|
698
757
|
----------
|
@@ -701,20 +760,20 @@ def serialize_geometry(
|
|
701
760
|
|
702
761
|
geometry_format: str, default 'wkb_hex'
|
703
762
|
The serialization format for geometry data.
|
704
|
-
Accepted formats are `wkb_hex`
|
705
|
-
`wkt` (well-known text), and `geojson`.
|
763
|
+
Accepted formats are `wkb`, `wkb_hex`, `wkt`, `geojson`, and `gpkg_wkb`.
|
706
764
|
|
707
765
|
srid: Optional[int], default None
|
708
766
|
If provided, use this as the source CRS when serializing to GeoJSON.
|
709
767
|
|
710
768
|
Returns
|
711
769
|
-------
|
712
|
-
A string containing the geometry data.
|
770
|
+
A string containing the geometry data, or bytes, or a dictionary, or None.
|
713
771
|
"""
|
714
772
|
if value_is_null(geom):
|
715
773
|
return None
|
716
|
-
|
717
|
-
|
774
|
+
|
775
|
+
shapely, shapely_ops, pyproj, np = mrsm.attempt_import(
|
776
|
+
'shapely', 'shapely.ops', 'pyproj', 'numpy',
|
718
777
|
lazy=False,
|
719
778
|
)
|
720
779
|
if geometry_format == 'geojson':
|
@@ -724,12 +783,31 @@ def serialize_geometry(
|
|
724
783
|
geojson_str = shapely.to_geojson(geom)
|
725
784
|
return json.loads(geojson_str)
|
726
785
|
|
727
|
-
if hasattr(geom, 'wkb_hex'):
|
728
|
-
|
729
|
-
|
730
|
-
|
786
|
+
if not hasattr(geom, 'wkb_hex'):
|
787
|
+
return str(geom)
|
788
|
+
|
789
|
+
byte_order = 1 if np.little_endian else 0
|
790
|
+
|
791
|
+
if geometry_format.startswith("wkb"):
|
792
|
+
return shapely.to_wkb(geom, hex=(geometry_format=="wkb_hex"), include_srid=True)
|
793
|
+
elif geometry_format == 'gpkg_wkb':
|
794
|
+
wkb_data = shapely.to_wkb(geom, hex=False, byte_order=byte_order)
|
795
|
+
flags = (
|
796
|
+
((byte_order & 0x01) | (0x20))
|
797
|
+
if geom.is_empty
|
798
|
+
else (byte_order & 0x01)
|
799
|
+
)
|
800
|
+
srid_val = srid or -1
|
801
|
+
header = struct.pack(
|
802
|
+
'<ccBBi',
|
803
|
+
b'G', b'P',
|
804
|
+
0,
|
805
|
+
flags,
|
806
|
+
srid_val
|
807
|
+
)
|
808
|
+
return header + wkb_data
|
731
809
|
|
732
|
-
return
|
810
|
+
return shapely.to_wkt(geom)
|
733
811
|
|
734
812
|
|
735
813
|
def deserialize_geometry(geom_wkb: Union[str, bytes]):
|
meerschaum/utils/dtypes/sql.py
CHANGED
@@ -635,7 +635,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
635
635
|
'mssql': 'NVARCHAR(MAX)',
|
636
636
|
'oracle': 'NVARCHAR2(2000)',
|
637
637
|
'sqlite': 'TEXT',
|
638
|
-
'geopackage': '
|
638
|
+
'geopackage': 'BLOB',
|
639
639
|
'duckdb': 'TEXT',
|
640
640
|
'citus': 'TEXT',
|
641
641
|
'cockroachdb': 'TEXT',
|
@@ -651,7 +651,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
651
651
|
'mssql': 'NVARCHAR(MAX)',
|
652
652
|
'oracle': 'NVARCHAR2(2000)',
|
653
653
|
'sqlite': 'TEXT',
|
654
|
-
'geopackage': '
|
654
|
+
'geopackage': 'BLOB',
|
655
655
|
'duckdb': 'TEXT',
|
656
656
|
'citus': 'TEXT',
|
657
657
|
'cockroachdb': 'TEXT',
|
@@ -1133,7 +1133,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
1133
1133
|
'mssql': 'UnicodeText',
|
1134
1134
|
'oracle': 'UnicodeText',
|
1135
1135
|
'sqlite': 'UnicodeText',
|
1136
|
-
'geopackage': '
|
1136
|
+
'geopackage': 'LargeBinary',
|
1137
1137
|
'duckdb': 'UnicodeText',
|
1138
1138
|
'citus': 'UnicodeText',
|
1139
1139
|
'cockroachdb': 'UnicodeText',
|
meerschaum/utils/sql.py
CHANGED
@@ -2146,6 +2146,7 @@ def get_create_table_queries(
|
|
2146
2146
|
primary_key_db_type: Optional[str] = None,
|
2147
2147
|
autoincrement: bool = False,
|
2148
2148
|
datetime_column: Optional[str] = None,
|
2149
|
+
_parse_dtypes: bool = True,
|
2149
2150
|
) -> List[str]:
|
2150
2151
|
"""
|
2151
2152
|
Return a query to create a new table from a `SELECT` query or a `dtypes` dictionary.
|
@@ -2179,6 +2180,10 @@ def get_create_table_queries(
|
|
2179
2180
|
If provided, include this column in the primary key.
|
2180
2181
|
Applicable to TimescaleDB only.
|
2181
2182
|
|
2183
|
+
_parse_dtypes: bool, default True
|
2184
|
+
If `True`, cast Pandas dtypes to SQL dtypes.
|
2185
|
+
Otherwise pass through the given value directly.
|
2186
|
+
|
2182
2187
|
Returns
|
2183
2188
|
-------
|
2184
2189
|
A `CREATE TABLE` (or `SELECT INTO`) query for the database flavor.
|
@@ -2200,6 +2205,7 @@ def get_create_table_queries(
|
|
2200
2205
|
primary_key_db_type=primary_key_db_type,
|
2201
2206
|
autoincrement=(autoincrement and flavor not in SKIP_AUTO_INCREMENT_FLAVORS),
|
2202
2207
|
datetime_column=datetime_column,
|
2208
|
+
_parse_dtypes=_parse_dtypes,
|
2203
2209
|
)
|
2204
2210
|
|
2205
2211
|
|
@@ -2212,6 +2218,7 @@ def _get_create_table_query_from_dtypes(
|
|
2212
2218
|
primary_key_db_type: Optional[str] = None,
|
2213
2219
|
autoincrement: bool = False,
|
2214
2220
|
datetime_column: Optional[str] = None,
|
2221
|
+
_parse_dtypes: bool = True,
|
2215
2222
|
) -> List[str]:
|
2216
2223
|
"""
|
2217
2224
|
Create a new table from a `dtypes` dictionary.
|
@@ -2228,12 +2235,22 @@ def _get_create_table_query_from_dtypes(
|
|
2228
2235
|
(
|
2229
2236
|
primary_key,
|
2230
2237
|
get_db_type_from_pd_type(dtypes.get(primary_key, 'int') or 'int', flavor=flavor)
|
2238
|
+
) if _parse_dtypes else (
|
2239
|
+
primary_key,
|
2240
|
+
dtypes.get(primary_key, 'INT') or 'INT'
|
2231
2241
|
)
|
2232
2242
|
]
|
2233
2243
|
if primary_key
|
2234
2244
|
else []
|
2235
2245
|
) + [
|
2236
|
-
(
|
2246
|
+
(
|
2247
|
+
col,
|
2248
|
+
(
|
2249
|
+
get_db_type_from_pd_type(typ, flavor=flavor)
|
2250
|
+
if _parse_dtypes
|
2251
|
+
else typ
|
2252
|
+
)
|
2253
|
+
)
|
2237
2254
|
for col, typ in dtypes.items()
|
2238
2255
|
if col != primary_key
|
2239
2256
|
]
|
@@ -2311,6 +2328,7 @@ def _get_create_table_query_from_cte(
|
|
2311
2328
|
primary_key_db_type: Optional[str] = None,
|
2312
2329
|
autoincrement: bool = False,
|
2313
2330
|
datetime_column: Optional[str] = None,
|
2331
|
+
_parse_dtypes=None,
|
2314
2332
|
) -> List[str]:
|
2315
2333
|
"""
|
2316
2334
|
Create a new table from a CTE query.
|
@@ -24,7 +24,7 @@ meerschaum/_internal/term/tools.py,sha256=8ocptBrvynEJ6iGbvcAzkq4TudaUJmAJ8c19Nw
|
|
24
24
|
meerschaum/actions/__init__.py,sha256=tR8Id90GMKF5L9QQwWtONtt7MZWCUbZ3NkE6J4ENeGM,11730
|
25
25
|
meerschaum/actions/api.py,sha256=7mQSSXkYIqetjOAiAGYHOjpzJSx4MN99ts6JwHlrNTc,14638
|
26
26
|
meerschaum/actions/attach.py,sha256=Vy41f8alMp3P-lXzyQkHqyDSDamtYGe2THWjFFrAdf4,3107
|
27
|
-
meerschaum/actions/bootstrap.py,sha256=
|
27
|
+
meerschaum/actions/bootstrap.py,sha256=gt9-IE_9VTNcY4g20cn3fbZ4NIJylA2b9-8A_wCeXD8,18489
|
28
28
|
meerschaum/actions/clear.py,sha256=v_xHn7-Pu7iwFNJ07q9eJt2hqPV7OwNZHUYa9dvixs4,4976
|
29
29
|
meerschaum/actions/copy.py,sha256=PlYXhtkRroReAI0wHHqevb0a4nTJOKgcVkEGASdVJNk,7000
|
30
30
|
meerschaum/actions/deduplicate.py,sha256=puYyxeFYEUy1Sd2IOcZB2e6MrNxAZl2bTLmNzFDkCiw,1167
|
@@ -144,7 +144,7 @@ meerschaum/api/routes/_webterm.py,sha256=TviHMz0_pLnby7dZW76y3mt-40Cq3VGFItSVcBi
|
|
144
144
|
meerschaum/api/tables/__init__.py,sha256=e2aNC0CdlWICTUMx1i9RauF8Pm426J0RZJbsJWv4SWo,482
|
145
145
|
meerschaum/config/__init__.py,sha256=IZLzkI7jBycfmHSR6R_WG3eBe0Uv53c-CI_QqNf4lnw,12966
|
146
146
|
meerschaum/config/_dash.py,sha256=BJHl4xMrQB-YHUEU7ldEW8q_nOPoIRSOqLrfGElc6Dw,187
|
147
|
-
meerschaum/config/_default.py,sha256=
|
147
|
+
meerschaum/config/_default.py,sha256=j5jeTzm7UwMjhIL_2Liq3UieVVeePpfZiMOS5HEmBMk,8134
|
148
148
|
meerschaum/config/_edit.py,sha256=WCdt7BSspPytH-NiPS1kpCZxXDDG7VbOu0H1AnMtwlw,9177
|
149
149
|
meerschaum/config/_formatting.py,sha256=SXUU0QFAlO_eIZjyodXQGSrjg-mVkL2qXuQwxqqdJps,6746
|
150
150
|
meerschaum/config/_jobs.py,sha256=gS_4mMGdmVP7WB4V5Sz8kYP0HmhWcMY2jSWGR7jX6cw,1279
|
@@ -154,7 +154,7 @@ meerschaum/config/_preprocess.py,sha256=-AEA8m_--KivZwTQ1sWN6LTn5sio_fUr2XZ51BO6
|
|
154
154
|
meerschaum/config/_read_config.py,sha256=Y4O-eozH94FsucqLWXb5hS57RLPMHsLUQe7a24doi0o,16231
|
155
155
|
meerschaum/config/_shell.py,sha256=46_m49Txc5q1rGfCgO49ca48BODx45DQJi8D0zz1R18,4245
|
156
156
|
meerschaum/config/_sync.py,sha256=nN5bLCHU8sFDdlPi7pQXuRVFcX457rZjOiALTvqRS_E,4332
|
157
|
-
meerschaum/config/_version.py,sha256=
|
157
|
+
meerschaum/config/_version.py,sha256=atbsLgB6KVm9tKp3EmV7gGIaBgwrSJ3-JJ4O9-RXWM4,71
|
158
158
|
meerschaum/config/environment.py,sha256=hTyjPLrGp3R4sytg_Jqv4TnCVI1WdmbX2YoRBCWSNt8,8277
|
159
159
|
meerschaum/config/paths.py,sha256=JjibeGN3YAdSNceRwsd42aNmeUrIgM6ndzC8qZAmNI0,621
|
160
160
|
meerschaum/config/static.py,sha256=92fSGxoHFDOK9GEsN8NVbAEh9W7-eZ1BS6thyEqcjeI,359
|
@@ -189,15 +189,15 @@ meerschaum/connectors/instance/_tokens.py,sha256=PejT6UEhkxkOCuEQf3Skvh0vTKTtE7c
|
|
189
189
|
meerschaum/connectors/instance/_users.py,sha256=NijU5wrBLjc67OUeJkyUNZYMC8KtAI-2ALFt36inT9g,5539
|
190
190
|
meerschaum/connectors/plugin/PluginConnector.py,sha256=aQ1QaB7MordCFimZqoGLb0R12PfDUN_nWks2J5mzeAs,2084
|
191
191
|
meerschaum/connectors/plugin/__init__.py,sha256=pwF7TGY4WNz2_HaVdmK4rPQ9ZwTOEuPHgzOqsGcoXJw,198
|
192
|
-
meerschaum/connectors/sql/_SQLConnector.py,sha256=
|
192
|
+
meerschaum/connectors/sql/_SQLConnector.py,sha256=NfjFrMI6W3C44q_BgcxVHSHoDjPP7Fj44l2sKGjbZCE,12980
|
193
193
|
meerschaum/connectors/sql/__init__.py,sha256=3cqYiDkVasn7zWdtOTAZbT4bo95AuvGOmDD2TkaAxtw,205
|
194
194
|
meerschaum/connectors/sql/_cli.py,sha256=OTMmiOLgRzQn2F_CXsIry4oWhvBek_D0wsX486Fq9jo,5345
|
195
195
|
meerschaum/connectors/sql/_create_engine.py,sha256=yTimP668KAmnA6p8AK4q2OnColwahpRahsluByEgVXU,7725
|
196
196
|
meerschaum/connectors/sql/_fetch.py,sha256=ZqtYd1LSGmz-UVXLcEppRAhY2mAUNAkE2SU-uubwRfk,12154
|
197
197
|
meerschaum/connectors/sql/_instance.py,sha256=xCc8M0xWMzF5Tu_1uWIFivAoHey5N1ccFhN_Z7u04zk,6304
|
198
|
-
meerschaum/connectors/sql/_pipes.py,sha256=
|
198
|
+
meerschaum/connectors/sql/_pipes.py,sha256=_tDpieGftfq_po3j6NR1ccKy6N_yDCoBOhvgFiEdL14,138182
|
199
199
|
meerschaum/connectors/sql/_plugins.py,sha256=SubR5HUJaetoUCv83YNEMwhv4wTTBCMcxSOEUgyMML4,8747
|
200
|
-
meerschaum/connectors/sql/_sql.py,sha256=
|
200
|
+
meerschaum/connectors/sql/_sql.py,sha256=troSThRxH7y-8XzUy_5EK7KVZT775-LxSKzw6445bBY,44647
|
201
201
|
meerschaum/connectors/sql/_uri.py,sha256=pSqGo6DfSvCUl2rjk833W3Foi20goZ0301pyqZR4R2I,3454
|
202
202
|
meerschaum/connectors/sql/_users.py,sha256=n3bW01GHWNT5QpI_uGyQV6aZA_FEwgV61QrhFmgNit8,10837
|
203
203
|
meerschaum/connectors/sql/tools.py,sha256=jz8huOaRCwGlYdtGfAqAh7SoK8uydYBrasKQba9FT38,187
|
@@ -256,7 +256,7 @@ meerschaum/utils/pool.py,sha256=vkE42af4fjrTEJTxf6Ek3xGucm1MtEkpsSEiaVzNKHs,2655
|
|
256
256
|
meerschaum/utils/process.py,sha256=Y7yk7b-OEADd28E40ffvIGLS3shi4-e604YWX5aFUDo,8214
|
257
257
|
meerschaum/utils/prompt.py,sha256=NmbSRJyrapH7UKMV1Ry_LIjj2POVTpNCLS2QeIogge0,19740
|
258
258
|
meerschaum/utils/schedule.py,sha256=TMUjQ57UFStfZfQk7GdQItDHrH888O178vum1ID_Drk,11454
|
259
|
-
meerschaum/utils/sql.py,sha256=
|
259
|
+
meerschaum/utils/sql.py,sha256=ydl-6ILfKUJ6BMXC3qLOC92p-I80pB9jG9U_2r6KVDA,89142
|
260
260
|
meerschaum/utils/threading.py,sha256=FV6mtQecZNmXJgirMcElX_6S0UXVaQHqhC4vzfqarbs,4022
|
261
261
|
meerschaum/utils/typing.py,sha256=LEoM4LbAbGIArBAecdRrtiMfrP8ctm9idjqCGJszVi4,2772
|
262
262
|
meerschaum/utils/warnings.py,sha256=voneQTz4OHFaL5JJ0Hu1lZS_8DrhJJfomPIvnFGq6pQ,6521
|
@@ -267,8 +267,8 @@ meerschaum/utils/daemon/RotatingFile.py,sha256=8KlA0llydMBoeBRpYyyy7BZSjcaS6ubDI
|
|
267
267
|
meerschaum/utils/daemon/StdinFile.py,sha256=jzJTqo36-XPVjzPCctvY_nHbhAUjLk7wN1Eq6gvvP3E,4771
|
268
268
|
meerschaum/utils/daemon/__init__.py,sha256=TM2U2m84s7Y80xHJ8BAIQOmGJc7XwjQNXvpQK-SNIJo,9587
|
269
269
|
meerschaum/utils/daemon/_names.py,sha256=DM98kaw6Th-ZO_3eKeysFoyCaod_q3gRS4ig_QPDT7k,4520
|
270
|
-
meerschaum/utils/dtypes/__init__.py,sha256=
|
271
|
-
meerschaum/utils/dtypes/sql.py,sha256=
|
270
|
+
meerschaum/utils/dtypes/__init__.py,sha256=ulsBlRAoZC7nw9hdwShVHqj-j0a5ThQkguaAqTbWXAc,38074
|
271
|
+
meerschaum/utils/dtypes/sql.py,sha256=FS6FiDB7USJe3NKaAvgEYkq7r-pwpsRHxdCNJRtlYa4,45698
|
272
272
|
meerschaum/utils/formatting/__init__.py,sha256=gP-vHE6iqyqdrTHXVjKc5ML5lUz2MFwaJYwLilduIUE,15585
|
273
273
|
meerschaum/utils/formatting/_jobs.py,sha256=c2Kp51vRJkDs2OeYLiePjSJCYhWj5az4b8uGodhG2a0,6637
|
274
274
|
meerschaum/utils/formatting/_pipes.py,sha256=CbCgUQadFR7i9gKogTPQ7vCgFguy9BW_hIT_aIwPhCw,19565
|
@@ -279,11 +279,11 @@ meerschaum/utils/packages/_packages.py,sha256=KWVfxsm-6ZdffpskIjHPki6wSQvf0LK9MW
|
|
279
279
|
meerschaum/utils/packages/lazy_loader.py,sha256=VHnph3VozH29R4JnSSBfwtA5WKZYZQFT_GeQSShCnuc,2540
|
280
280
|
meerschaum/utils/venv/_Venv.py,sha256=lYiuyCloG1OmKh_8xMMXHkAq1VIMUWLhYGtCVm6nvdA,4044
|
281
281
|
meerschaum/utils/venv/__init__.py,sha256=P40aC-RMz3iGrMnED6D4tha6Tj0Mz-pDgUqJVibgjwo,27767
|
282
|
-
meerschaum-3.0.
|
283
|
-
meerschaum-3.0.
|
284
|
-
meerschaum-3.0.
|
285
|
-
meerschaum-3.0.
|
286
|
-
meerschaum-3.0.
|
287
|
-
meerschaum-3.0.
|
288
|
-
meerschaum-3.0.
|
289
|
-
meerschaum-3.0.
|
282
|
+
meerschaum-3.0.9.dist-info/licenses/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
|
283
|
+
meerschaum-3.0.9.dist-info/licenses/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
|
284
|
+
meerschaum-3.0.9.dist-info/METADATA,sha256=jXNtKURr_b90gZc1dPAOasPOYQus_2fSG7tAlRSrGPY,25325
|
285
|
+
meerschaum-3.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
286
|
+
meerschaum-3.0.9.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
|
287
|
+
meerschaum-3.0.9.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
|
288
|
+
meerschaum-3.0.9.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
289
|
+
meerschaum-3.0.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|