meerschaum 2.8.4__py3-none-any.whl → 2.9.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/api/_chunks.py +67 -0
- meerschaum/api/dash/callbacks/custom.py +23 -2
- meerschaum/api/dash/callbacks/dashboard.py +41 -3
- meerschaum/api/dash/components.py +27 -19
- meerschaum/api/dash/pages/dashboard.py +11 -9
- meerschaum/api/dash/pages/plugins.py +31 -27
- meerschaum/api/dash/webterm.py +6 -3
- meerschaum/api/resources/static/css/dash.css +1 -1
- meerschaum/api/resources/templates/termpage.html +4 -0
- meerschaum/api/routes/_pipes.py +191 -78
- meerschaum/config/_default.py +4 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_APIConnector.py +12 -1
- meerschaum/connectors/api/_pipes.py +27 -15
- meerschaum/connectors/api/_plugins.py +51 -45
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/parse.py +1 -2
- meerschaum/connectors/sql/_SQLConnector.py +3 -0
- meerschaum/connectors/sql/_cli.py +1 -0
- meerschaum/connectors/sql/_create_engine.py +51 -4
- meerschaum/connectors/sql/_pipes.py +13 -2
- meerschaum/connectors/sql/_sql.py +35 -4
- meerschaum/core/Pipe/_data.py +1 -2
- meerschaum/plugins/_Plugin.py +21 -5
- meerschaum/plugins/__init__.py +6 -4
- meerschaum/utils/dataframe.py +87 -2
- meerschaum/utils/dtypes/__init__.py +182 -1
- meerschaum/utils/dtypes/sql.py +114 -2
- meerschaum/utils/formatting/_shell.py +1 -4
- meerschaum/utils/packages/_packages.py +3 -0
- meerschaum/utils/sql.py +17 -5
- meerschaum/utils/venv/__init__.py +2 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/METADATA +10 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/RECORD +40 -39
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/WHEEL +1 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/LICENSE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/NOTICE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0rc1.dist-info}/zip-safe +0 -0
meerschaum/utils/dtypes/sql.py
CHANGED
@@ -128,6 +128,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
128
128
|
'int': {
|
129
129
|
'timescaledb': 'BIGINT',
|
130
130
|
'postgresql': 'BIGINT',
|
131
|
+
'postgis': 'BIGINT',
|
131
132
|
'mariadb': 'BIGINT',
|
132
133
|
'mysql': 'BIGINT',
|
133
134
|
'mssql': 'BIGINT',
|
@@ -141,6 +142,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
141
142
|
'float': {
|
142
143
|
'timescaledb': 'DOUBLE PRECISION',
|
143
144
|
'postgresql': 'DOUBLE PRECISION',
|
145
|
+
'postgis': 'DOUBLE PRECISION',
|
144
146
|
'mariadb': 'DOUBLE PRECISION',
|
145
147
|
'mysql': 'DOUBLE PRECISION',
|
146
148
|
'mssql': 'FLOAT',
|
@@ -154,6 +156,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
154
156
|
'double': {
|
155
157
|
'timescaledb': 'DOUBLE PRECISION',
|
156
158
|
'postgresql': 'DOUBLE PRECISION',
|
159
|
+
'postgis': 'DOUBLE PRECISION',
|
157
160
|
'mariadb': 'DOUBLE PRECISION',
|
158
161
|
'mysql': 'DOUBLE PRECISION',
|
159
162
|
'mssql': 'FLOAT',
|
@@ -167,6 +170,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
167
170
|
'datetime64[ns]': {
|
168
171
|
'timescaledb': 'TIMESTAMP',
|
169
172
|
'postgresql': 'TIMESTAMP',
|
173
|
+
'postgis': 'TIMESTAMP',
|
170
174
|
'mariadb': 'DATETIME',
|
171
175
|
'mysql': 'DATETIME',
|
172
176
|
'mssql': 'DATETIME2',
|
@@ -180,6 +184,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
180
184
|
'datetime64[ns, UTC]': {
|
181
185
|
'timescaledb': 'TIMESTAMPTZ',
|
182
186
|
'postgresql': 'TIMESTAMPTZ',
|
187
|
+
'postgis': 'TIMESTAMPTZ',
|
183
188
|
'mariadb': 'DATETIME',
|
184
189
|
'mysql': 'DATETIME',
|
185
190
|
'mssql': 'DATETIMEOFFSET',
|
@@ -193,6 +198,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
193
198
|
'datetime': {
|
194
199
|
'timescaledb': 'TIMESTAMPTZ',
|
195
200
|
'postgresql': 'TIMESTAMPTZ',
|
201
|
+
'postgis': 'TIMESTAMPTZ',
|
196
202
|
'mariadb': 'DATETIME',
|
197
203
|
'mysql': 'DATETIME',
|
198
204
|
'mssql': 'DATETIMEOFFSET',
|
@@ -206,6 +212,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
206
212
|
'datetimetz': {
|
207
213
|
'timescaledb': 'TIMESTAMPTZ',
|
208
214
|
'postgresql': 'TIMESTAMPTZ',
|
215
|
+
'postgis': 'TIMESTAMPTZ',
|
209
216
|
'mariadb': 'DATETIME',
|
210
217
|
'mysql': 'DATETIME',
|
211
218
|
'mssql': 'DATETIMEOFFSET',
|
@@ -219,6 +226,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
219
226
|
'bool': {
|
220
227
|
'timescaledb': 'BOOLEAN',
|
221
228
|
'postgresql': 'BOOLEAN',
|
229
|
+
'postgis': 'BOOLEAN',
|
222
230
|
'mariadb': 'BOOLEAN',
|
223
231
|
'mysql': 'BOOLEAN',
|
224
232
|
'mssql': 'BIT',
|
@@ -232,6 +240,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
232
240
|
'object': {
|
233
241
|
'timescaledb': 'TEXT',
|
234
242
|
'postgresql': 'TEXT',
|
243
|
+
'postgis': 'TEXT',
|
235
244
|
'mariadb': 'TEXT',
|
236
245
|
'mysql': 'TEXT',
|
237
246
|
'mssql': 'NVARCHAR(MAX)',
|
@@ -245,6 +254,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
245
254
|
'string': {
|
246
255
|
'timescaledb': 'TEXT',
|
247
256
|
'postgresql': 'TEXT',
|
257
|
+
'postgis': 'TEXT',
|
248
258
|
'mariadb': 'TEXT',
|
249
259
|
'mysql': 'TEXT',
|
250
260
|
'mssql': 'NVARCHAR(MAX)',
|
@@ -258,6 +268,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
258
268
|
'unicode': {
|
259
269
|
'timescaledb': 'TEXT',
|
260
270
|
'postgresql': 'TEXT',
|
271
|
+
'postgis': 'TEXT',
|
261
272
|
'mariadb': 'TEXT',
|
262
273
|
'mysql': 'TEXT',
|
263
274
|
'mssql': 'NVARCHAR(MAX)',
|
@@ -271,6 +282,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
271
282
|
'json': {
|
272
283
|
'timescaledb': 'JSONB',
|
273
284
|
'postgresql': 'JSONB',
|
285
|
+
'postgis': 'JSONB',
|
274
286
|
'mariadb': 'TEXT',
|
275
287
|
'mysql': 'TEXT',
|
276
288
|
'mssql': 'NVARCHAR(MAX)',
|
@@ -284,6 +296,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
284
296
|
'numeric': {
|
285
297
|
'timescaledb': 'NUMERIC',
|
286
298
|
'postgresql': 'NUMERIC',
|
299
|
+
'postgis': 'NUMERIC',
|
287
300
|
'mariadb': f'DECIMAL{NUMERIC_PRECISION_FLAVORS["mariadb"]}',
|
288
301
|
'mysql': f'DECIMAL{NUMERIC_PRECISION_FLAVORS["mysql"]}',
|
289
302
|
'mssql': f'NUMERIC{NUMERIC_PRECISION_FLAVORS["mssql"]}',
|
@@ -297,6 +310,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
297
310
|
'uuid': {
|
298
311
|
'timescaledb': 'UUID',
|
299
312
|
'postgresql': 'UUID',
|
313
|
+
'postgis': 'UUID',
|
300
314
|
'mariadb': 'CHAR(36)',
|
301
315
|
'mysql': 'CHAR(36)',
|
302
316
|
'mssql': 'UNIQUEIDENTIFIER',
|
@@ -311,6 +325,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
311
325
|
'bytes': {
|
312
326
|
'timescaledb': 'BYTEA',
|
313
327
|
'postgresql': 'BYTEA',
|
328
|
+
'postgis': 'BYTEA',
|
314
329
|
'mariadb': 'BLOB',
|
315
330
|
'mysql': 'BLOB',
|
316
331
|
'mssql': 'VARBINARY(MAX)',
|
@@ -321,11 +336,40 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
321
336
|
'cockroachdb': 'BYTEA',
|
322
337
|
'default': 'BLOB',
|
323
338
|
},
|
339
|
+
'geometry': {
|
340
|
+
'timescaledb': 'TEXT',
|
341
|
+
'postgresql': 'TEXT',
|
342
|
+
'postgis': 'GEOMETRY',
|
343
|
+
'mariadb': 'TEXT',
|
344
|
+
'mysql': 'TEXT',
|
345
|
+
'mssql': 'NVARCHAR(MAX)',
|
346
|
+
'oracle': 'NVARCHAR2(2000)',
|
347
|
+
'sqlite': 'TEXT',
|
348
|
+
'duckdb': 'TEXT',
|
349
|
+
'citus': 'TEXT',
|
350
|
+
'cockroachdb': 'TEXT',
|
351
|
+
'default': 'TEXT',
|
352
|
+
},
|
353
|
+
'geography': {
|
354
|
+
'timescaledb': 'TEXT',
|
355
|
+
'postgresql': 'TEXT',
|
356
|
+
'postgis': 'GEOGRAPHY',
|
357
|
+
'mariadb': 'TEXT',
|
358
|
+
'mysql': 'TEXT',
|
359
|
+
'mssql': 'NVARCHAR(MAX)',
|
360
|
+
'oracle': 'NVARCHAR2(2000)',
|
361
|
+
'sqlite': 'TEXT',
|
362
|
+
'duckdb': 'TEXT',
|
363
|
+
'citus': 'TEXT',
|
364
|
+
'cockroachdb': 'TEXT',
|
365
|
+
'default': 'TEXT',
|
366
|
+
},
|
324
367
|
}
|
325
368
|
PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
326
369
|
'int': {
|
327
370
|
'timescaledb': 'BigInteger',
|
328
371
|
'postgresql': 'BigInteger',
|
372
|
+
'postgis': 'BigInteger',
|
329
373
|
'mariadb': 'BigInteger',
|
330
374
|
'mysql': 'BigInteger',
|
331
375
|
'mssql': 'BigInteger',
|
@@ -339,6 +383,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
339
383
|
'float': {
|
340
384
|
'timescaledb': 'Float',
|
341
385
|
'postgresql': 'Float',
|
386
|
+
'postgis': 'Float',
|
342
387
|
'mariadb': 'Float',
|
343
388
|
'mysql': 'Float',
|
344
389
|
'mssql': 'Float',
|
@@ -352,6 +397,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
352
397
|
'datetime': {
|
353
398
|
'timescaledb': 'DateTime(timezone=True)',
|
354
399
|
'postgresql': 'DateTime(timezone=True)',
|
400
|
+
'postgis': 'DateTime(timezone=True)',
|
355
401
|
'mariadb': 'DateTime(timezone=True)',
|
356
402
|
'mysql': 'DateTime(timezone=True)',
|
357
403
|
'mssql': 'sqlalchemy.dialects.mssql.DATETIMEOFFSET',
|
@@ -365,6 +411,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
365
411
|
'datetime64[ns]': {
|
366
412
|
'timescaledb': 'DateTime',
|
367
413
|
'postgresql': 'DateTime',
|
414
|
+
'postgis': 'DateTime',
|
368
415
|
'mariadb': 'DateTime',
|
369
416
|
'mysql': 'DateTime',
|
370
417
|
'mssql': 'sqlalchemy.dialects.mssql.DATETIME2',
|
@@ -378,6 +425,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
378
425
|
'datetime64[ns, UTC]': {
|
379
426
|
'timescaledb': 'DateTime(timezone=True)',
|
380
427
|
'postgresql': 'DateTime(timezone=True)',
|
428
|
+
'postgis': 'DateTime(timezone=True)',
|
381
429
|
'mariadb': 'DateTime(timezone=True)',
|
382
430
|
'mysql': 'DateTime(timezone=True)',
|
383
431
|
'mssql': 'sqlalchemy.dialects.mssql.DATETIMEOFFSET',
|
@@ -391,6 +439,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
391
439
|
'bool': {
|
392
440
|
'timescaledb': 'Boolean',
|
393
441
|
'postgresql': 'Boolean',
|
442
|
+
'postgis': 'Boolean',
|
394
443
|
'mariadb': 'Integer',
|
395
444
|
'mysql': 'Integer',
|
396
445
|
'mssql': 'sqlalchemy.dialects.mssql.BIT',
|
@@ -404,6 +453,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
404
453
|
'object': {
|
405
454
|
'timescaledb': 'UnicodeText',
|
406
455
|
'postgresql': 'UnicodeText',
|
456
|
+
'postgis': 'UnicodeText',
|
407
457
|
'mariadb': 'UnicodeText',
|
408
458
|
'mysql': 'UnicodeText',
|
409
459
|
'mssql': 'UnicodeText',
|
@@ -417,6 +467,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
417
467
|
'string': {
|
418
468
|
'timescaledb': 'UnicodeText',
|
419
469
|
'postgresql': 'UnicodeText',
|
470
|
+
'postgis': 'UnicodeText',
|
420
471
|
'mariadb': 'UnicodeText',
|
421
472
|
'mysql': 'UnicodeText',
|
422
473
|
'mssql': 'UnicodeText',
|
@@ -430,6 +481,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
430
481
|
'json': {
|
431
482
|
'timescaledb': 'sqlalchemy.dialects.postgresql.JSONB',
|
432
483
|
'postgresql': 'sqlalchemy.dialects.postgresql.JSONB',
|
484
|
+
'postgis': 'sqlalchemy.dialects.postgresql.JSONB',
|
433
485
|
'mariadb': 'UnicodeText',
|
434
486
|
'mysql': 'UnicodeText',
|
435
487
|
'mssql': 'UnicodeText',
|
@@ -443,6 +495,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
443
495
|
'numeric': {
|
444
496
|
'timescaledb': 'Numeric',
|
445
497
|
'postgresql': 'Numeric',
|
498
|
+
'postgis': 'Numeric',
|
446
499
|
'mariadb': 'Numeric',
|
447
500
|
'mysql': 'Numeric',
|
448
501
|
'mssql': 'Numeric',
|
@@ -456,6 +509,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
456
509
|
'uuid': {
|
457
510
|
'timescaledb': 'Uuid',
|
458
511
|
'postgresql': 'Uuid',
|
512
|
+
'postgis': 'Uuid',
|
459
513
|
'mariadb': 'sqlalchemy.dialects.mysql.CHAR(36)',
|
460
514
|
'mysql': 'sqlalchemy.dialects.mysql.CHAR(36)',
|
461
515
|
'mssql': 'Uuid',
|
@@ -469,6 +523,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
469
523
|
'bytes': {
|
470
524
|
'timescaledb': 'LargeBinary',
|
471
525
|
'postgresql': 'LargeBinary',
|
526
|
+
'postgis': 'LargeBinary',
|
472
527
|
'mariadb': 'LargeBinary',
|
473
528
|
'mysql': 'LargeBinary',
|
474
529
|
'mssql': 'LargeBinary',
|
@@ -479,11 +534,40 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
479
534
|
'cockroachdb': 'LargeBinary',
|
480
535
|
'default': 'LargeBinary',
|
481
536
|
},
|
537
|
+
'geometry': {
|
538
|
+
'timescaledb': 'UnicodeText',
|
539
|
+
'postgresql': 'UnicodeText',
|
540
|
+
'postgis': 'geoalchemy2.Geometry',
|
541
|
+
'mariadb': 'UnicodeText',
|
542
|
+
'mysql': 'UnicodeText',
|
543
|
+
'mssql': 'UnicodeText',
|
544
|
+
'oracle': 'UnicodeText',
|
545
|
+
'sqlite': 'UnicodeText',
|
546
|
+
'duckdb': 'UnicodeText',
|
547
|
+
'citus': 'UnicodeText',
|
548
|
+
'cockroachdb': 'UnicodeText',
|
549
|
+
'default': 'UnicodeText',
|
550
|
+
},
|
551
|
+
'geography': {
|
552
|
+
'timescaledb': 'UnicodeText',
|
553
|
+
'postgresql': 'UnicodeText',
|
554
|
+
'postgis': 'geoalchemy2.Geography',
|
555
|
+
'mariadb': 'UnicodeText',
|
556
|
+
'mysql': 'UnicodeText',
|
557
|
+
'mssql': 'UnicodeText',
|
558
|
+
'oracle': 'UnicodeText',
|
559
|
+
'sqlite': 'UnicodeText',
|
560
|
+
'duckdb': 'UnicodeText',
|
561
|
+
'citus': 'UnicodeText',
|
562
|
+
'cockroachdb': 'UnicodeText',
|
563
|
+
'default': 'UnicodeText',
|
564
|
+
},
|
482
565
|
}
|
483
566
|
|
484
567
|
AUTO_INCREMENT_COLUMN_FLAVORS: Dict[str, str] = {
|
485
568
|
'timescaledb': 'GENERATED BY DEFAULT AS IDENTITY',
|
486
569
|
'postgresql': 'GENERATED BY DEFAULT AS IDENTITY',
|
570
|
+
'postgis': 'GENERATED BY DEFAULT AS IDENTITY',
|
487
571
|
'mariadb': 'AUTO_INCREMENT',
|
488
572
|
'mysql': 'AUTO_INCREMENT',
|
489
573
|
'mssql': 'IDENTITY(1,1)',
|
@@ -565,7 +649,7 @@ def get_db_type_from_pd_type(
|
|
565
649
|
"""
|
566
650
|
from meerschaum.utils.warnings import warn
|
567
651
|
from meerschaum.utils.packages import attempt_import
|
568
|
-
from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_ALIAS_DTYPES
|
652
|
+
from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_ALIAS_DTYPES, get_geometry_type_srid
|
569
653
|
from meerschaum.utils.misc import parse_arguments_str
|
570
654
|
sqlalchemy_types = attempt_import('sqlalchemy.types', lazy=False)
|
571
655
|
|
@@ -576,18 +660,30 @@ def get_db_type_from_pd_type(
|
|
576
660
|
)
|
577
661
|
|
578
662
|
precision, scale = None, None
|
663
|
+
geometry_type, geometry_srid = None, None
|
579
664
|
og_pd_type = pd_type
|
580
665
|
if pd_type in MRSM_ALIAS_DTYPES:
|
581
666
|
pd_type = MRSM_ALIAS_DTYPES[pd_type]
|
582
667
|
|
583
668
|
### Check whether we are able to match this type (e.g. pyarrow support).
|
584
669
|
found_db_type = False
|
585
|
-
if
|
670
|
+
if (
|
671
|
+
pd_type not in types_registry
|
672
|
+
and not any(
|
673
|
+
pd_type.startswith(f'{typ}[')
|
674
|
+
for typ in ('numeric', 'geometry', 'geography')
|
675
|
+
)
|
676
|
+
):
|
586
677
|
for mapped_pd_type in types_registry:
|
587
678
|
if are_dtypes_equal(mapped_pd_type, pd_type):
|
588
679
|
pd_type = mapped_pd_type
|
589
680
|
found_db_type = True
|
590
681
|
break
|
682
|
+
elif (pd_type.startswith('geometry[') or pd_type.startswith('geography[')):
|
683
|
+
og_pd_type = pd_type
|
684
|
+
pd_type = 'geometry' if 'geometry' in pd_type else 'geography'
|
685
|
+
geometry_type, geometry_srid = get_geometry_type_srid(og_pd_type)
|
686
|
+
found_db_type = True
|
591
687
|
elif pd_type.startswith('numeric['):
|
592
688
|
og_pd_type = pd_type
|
593
689
|
pd_type = 'numeric'
|
@@ -628,6 +724,11 @@ def get_db_type_from_pd_type(
|
|
628
724
|
if precision is not None and scale is not None:
|
629
725
|
db_type_bare = db_type.split('(', maxsplit=1)[0]
|
630
726
|
return f"{db_type_bare}({precision},{scale})"
|
727
|
+
if geometry_type is not None and geometry_srid is not None:
|
728
|
+
if 'geometry' not in db_type.lower() and 'geography' not in db_type.lower():
|
729
|
+
return db_type
|
730
|
+
db_type_bare = db_type.split('(', maxsplit=1)[0]
|
731
|
+
return f"{db_type_bare}({geometry_type.upper()}, {geometry_srid})"
|
631
732
|
return db_type
|
632
733
|
|
633
734
|
if db_type.startswith('sqlalchemy.dialects'):
|
@@ -643,6 +744,17 @@ def get_db_type_from_pd_type(
|
|
643
744
|
return cls
|
644
745
|
return cls(*cls_args, **cls_kwargs)
|
645
746
|
|
747
|
+
if 'geometry' in db_type.lower() or 'geography' in db_type.lower():
|
748
|
+
geoalchemy2 = attempt_import('geoalchemy2', lazy=False)
|
749
|
+
geometry_class = (
|
750
|
+
geoalchemy2.Geometry
|
751
|
+
if 'geometry' in db_type.lower()
|
752
|
+
else geoalchemy2.Geography
|
753
|
+
)
|
754
|
+
if geometry_type is None or geometry_srid is None:
|
755
|
+
return geometry_class
|
756
|
+
return geometry_class(geometry_type=geometry_type, srid=geometry_srid)
|
757
|
+
|
646
758
|
if 'numeric' in db_type.lower():
|
647
759
|
if precision is None or scale is None:
|
648
760
|
return sqlalchemy_types.Numeric
|
@@ -66,10 +66,7 @@ def clear_screen(debug: bool = False) -> bool:
|
|
66
66
|
get_console().clear()
|
67
67
|
print("", end="", flush=True)
|
68
68
|
return True
|
69
|
-
clear_string =
|
70
|
-
'\033c' if platform.system() != 'Windows'
|
71
|
-
else '\033[2J'
|
72
|
-
)
|
69
|
+
clear_string = '\033[2J'
|
73
70
|
reset_string = '\033[0m'
|
74
71
|
print(clear_string + reset_string, end="")
|
75
72
|
print("", end="", flush=True)
|
@@ -135,12 +135,15 @@ packages: Dict[str, Dict[str, str]] = {
|
|
135
135
|
packages['sql'] = {
|
136
136
|
'numpy' : 'numpy>=1.18.5',
|
137
137
|
'pandas' : 'pandas[parquet]>=2.0.1',
|
138
|
+
'geopandas' : 'geopandas>=1.0.1',
|
139
|
+
'shapely' : 'shapely>=2.0.7',
|
138
140
|
'pyarrow' : 'pyarrow>=16.1.0',
|
139
141
|
'dask' : 'dask[complete]>=2024.12.1',
|
140
142
|
'partd' : 'partd>=1.4.2',
|
141
143
|
'pytz' : 'pytz',
|
142
144
|
'joblib' : 'joblib>=0.17.0',
|
143
145
|
'sqlalchemy' : 'SQLAlchemy>=2.0.5',
|
146
|
+
'geoalchemy' : 'GeoAlchemy2>=0.17.1',
|
144
147
|
'databases' : 'databases>=0.4.0',
|
145
148
|
'aiosqlite' : 'aiosqlite>=0.16.0',
|
146
149
|
'asyncpg' : 'asyncpg>=0.21.0',
|
meerschaum/utils/sql.py
CHANGED
@@ -41,13 +41,13 @@ version_queries = {
|
|
41
41
|
}
|
42
42
|
SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
|
43
43
|
DROP_IF_EXISTS_FLAVORS = {
|
44
|
-
'timescaledb', 'postgresql', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
|
44
|
+
'timescaledb', 'postgresql', 'postgis', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
|
45
45
|
}
|
46
46
|
DROP_INDEX_IF_EXISTS_FLAVORS = {
|
47
|
-
'mssql', 'timescaledb', 'postgresql', 'sqlite', 'citus',
|
47
|
+
'mssql', 'timescaledb', 'postgresql', 'postgis', 'sqlite', 'citus',
|
48
48
|
}
|
49
49
|
SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
|
50
|
-
COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
|
50
|
+
COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'postgis', 'citus'}
|
51
51
|
UPDATE_QUERIES = {
|
52
52
|
'default': """
|
53
53
|
UPDATE {target_table_name} AS f
|
@@ -73,6 +73,12 @@ UPDATE_QUERIES = {
|
|
73
73
|
FROM {patch_table_name}
|
74
74
|
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
75
75
|
""",
|
76
|
+
'postgis-upsert': """
|
77
|
+
INSERT INTO {target_table_name} ({patch_cols_str})
|
78
|
+
SELECT {patch_cols_str}
|
79
|
+
FROM {patch_table_name}
|
80
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
81
|
+
""",
|
76
82
|
'citus-upsert': """
|
77
83
|
INSERT INTO {target_table_name} ({patch_cols_str})
|
78
84
|
SELECT {patch_cols_str}
|
@@ -482,6 +488,7 @@ table_wrappers = {
|
|
482
488
|
'citus' : ('"', '"'),
|
483
489
|
'duckdb' : ('"', '"'),
|
484
490
|
'postgresql' : ('"', '"'),
|
491
|
+
'postgis' : ('"', '"'),
|
485
492
|
'sqlite' : ('"', '"'),
|
486
493
|
'mysql' : ('`', '`'),
|
487
494
|
'mariadb' : ('`', '`'),
|
@@ -494,6 +501,7 @@ max_name_lens = {
|
|
494
501
|
'mssql' : 128,
|
495
502
|
'oracle' : 30,
|
496
503
|
'postgresql' : 64,
|
504
|
+
'postgis' : 64,
|
497
505
|
'timescaledb': 64,
|
498
506
|
'citus' : 64,
|
499
507
|
'cockroachdb': 64,
|
@@ -501,10 +509,11 @@ max_name_lens = {
|
|
501
509
|
'mysql' : 64,
|
502
510
|
'mariadb' : 64,
|
503
511
|
}
|
504
|
-
json_flavors = {'postgresql', 'timescaledb', 'citus', 'cockroachdb'}
|
512
|
+
json_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'cockroachdb'}
|
505
513
|
NO_SCHEMA_FLAVORS = {'oracle', 'sqlite', 'mysql', 'mariadb', 'duckdb'}
|
506
514
|
DEFAULT_SCHEMA_FLAVORS = {
|
507
515
|
'postgresql': 'public',
|
516
|
+
'postgis': 'public',
|
508
517
|
'timescaledb': 'public',
|
509
518
|
'citus': 'public',
|
510
519
|
'cockroachdb': 'public',
|
@@ -549,6 +558,7 @@ def dateadd_str(
|
|
549
558
|
Currently supported flavors:
|
550
559
|
|
551
560
|
- `'postgresql'`
|
561
|
+
- `'postgis'`
|
552
562
|
- `'timescaledb'`
|
553
563
|
- `'citus'`
|
554
564
|
- `'cockroachdb'`
|
@@ -653,7 +663,7 @@ def dateadd_str(
|
|
653
663
|
)
|
654
664
|
|
655
665
|
da = ""
|
656
|
-
if flavor in ('postgresql', 'timescaledb', 'cockroachdb', 'citus'):
|
666
|
+
if flavor in ('postgresql', 'postgis', 'timescaledb', 'cockroachdb', 'citus'):
|
657
667
|
begin = (
|
658
668
|
f"CAST({begin} AS {db_type})" if begin != 'now'
|
659
669
|
else f"CAST(NOW() AT TIME ZONE 'utc' AS {db_type})"
|
@@ -1809,6 +1819,8 @@ def get_null_replacement(typ: str, flavor: str) -> str:
|
|
1809
1819
|
"""
|
1810
1820
|
from meerschaum.utils.dtypes import are_dtypes_equal
|
1811
1821
|
from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
|
1822
|
+
if 'geometry' in typ.lower():
|
1823
|
+
return '010100000000008058346FCDC100008058346FCDC1'
|
1812
1824
|
if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
|
1813
1825
|
return '-987654321'
|
1814
1826
|
if 'bool' in typ.lower() or typ.lower() == 'bit':
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: meerschaum
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.9.0rc1
|
4
4
|
Summary: Sync Time-Series Pipes with Meerschaum
|
5
5
|
Home-page: https://meerschaum.io
|
6
6
|
Author: Bennett Meares
|
@@ -126,12 +126,15 @@ Requires-Dist: importlib-metadata>=4.12.0; extra == "extras"
|
|
126
126
|
Provides-Extra: sql
|
127
127
|
Requires-Dist: numpy>=1.18.5; extra == "sql"
|
128
128
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
|
129
|
+
Requires-Dist: geopandas>=1.0.1; extra == "sql"
|
130
|
+
Requires-Dist: shapely>=2.0.7; extra == "sql"
|
129
131
|
Requires-Dist: pyarrow>=16.1.0; extra == "sql"
|
130
132
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "sql"
|
131
133
|
Requires-Dist: partd>=1.4.2; extra == "sql"
|
132
134
|
Requires-Dist: pytz; extra == "sql"
|
133
135
|
Requires-Dist: joblib>=0.17.0; extra == "sql"
|
134
136
|
Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
|
137
|
+
Requires-Dist: GeoAlchemy2>=0.17.1; extra == "sql"
|
135
138
|
Requires-Dist: databases>=0.4.0; extra == "sql"
|
136
139
|
Requires-Dist: aiosqlite>=0.16.0; extra == "sql"
|
137
140
|
Requires-Dist: asyncpg>=0.21.0; extra == "sql"
|
@@ -186,12 +189,15 @@ Requires-Dist: httpcore>=1.0.6; extra == "api"
|
|
186
189
|
Requires-Dist: valkey>=6.0.0; extra == "api"
|
187
190
|
Requires-Dist: numpy>=1.18.5; extra == "api"
|
188
191
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
|
192
|
+
Requires-Dist: geopandas>=1.0.1; extra == "api"
|
193
|
+
Requires-Dist: shapely>=2.0.7; extra == "api"
|
189
194
|
Requires-Dist: pyarrow>=16.1.0; extra == "api"
|
190
195
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "api"
|
191
196
|
Requires-Dist: partd>=1.4.2; extra == "api"
|
192
197
|
Requires-Dist: pytz; extra == "api"
|
193
198
|
Requires-Dist: joblib>=0.17.0; extra == "api"
|
194
199
|
Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
|
200
|
+
Requires-Dist: GeoAlchemy2>=0.17.1; extra == "api"
|
195
201
|
Requires-Dist: databases>=0.4.0; extra == "api"
|
196
202
|
Requires-Dist: aiosqlite>=0.16.0; extra == "api"
|
197
203
|
Requires-Dist: asyncpg>=0.21.0; extra == "api"
|
@@ -291,12 +297,15 @@ Requires-Dist: pywebview>=3.6.3; extra == "full"
|
|
291
297
|
Requires-Dist: pycparser>=2.21.0; extra == "full"
|
292
298
|
Requires-Dist: numpy>=1.18.5; extra == "full"
|
293
299
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
|
300
|
+
Requires-Dist: geopandas>=1.0.1; extra == "full"
|
301
|
+
Requires-Dist: shapely>=2.0.7; extra == "full"
|
294
302
|
Requires-Dist: pyarrow>=16.1.0; extra == "full"
|
295
303
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "full"
|
296
304
|
Requires-Dist: partd>=1.4.2; extra == "full"
|
297
305
|
Requires-Dist: pytz; extra == "full"
|
298
306
|
Requires-Dist: joblib>=0.17.0; extra == "full"
|
299
307
|
Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"
|
308
|
+
Requires-Dist: GeoAlchemy2>=0.17.1; extra == "full"
|
300
309
|
Requires-Dist: databases>=0.4.0; extra == "full"
|
301
310
|
Requires-Dist: aiosqlite>=0.16.0; extra == "full"
|
302
311
|
Requires-Dist: asyncpg>=0.21.0; extra == "full"
|