meerschaum 2.6.0__py3-none-any.whl → 2.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,8 +7,16 @@ Register Pipes via the Meerschaum API.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import Any, Optional, Dict, Union
11
10
 
11
+
12
+ import io
13
+ import json
14
+ import fastapi
15
+ from decimal import Decimal
16
+ import datetime
17
+
18
+ import meerschaum as mrsm
19
+ from meerschaum.utils.typing import Any, Optional, Dict, Union, List
12
20
  from meerschaum.api import (
13
21
  fastapi,
14
22
  app,
@@ -19,27 +27,27 @@ from meerschaum.api import (
19
27
  _get_pipes,
20
28
  manager,
21
29
  debug,
22
- no_auth, private,
30
+ no_auth,
31
+ private,
23
32
  )
24
- import json
25
- import fastapi
26
- from decimal import Decimal
27
33
  from meerschaum import Pipe
28
34
  from meerschaum.api.models import MetaPipe
29
35
  from meerschaum.utils.packages import attempt_import, import_pandas
30
- from meerschaum.utils.dataframe import get_numeric_cols, to_json
36
+ from meerschaum.utils.dataframe import get_numeric_cols, to_json, parse_df_datetimes
37
+ from meerschaum.utils.dtypes import are_dtypes_equal
31
38
  from meerschaum.utils.misc import (
32
- is_pipe_registered, round_time, is_int, parse_df_datetimes,
39
+ is_pipe_registered,
40
+ round_time,
41
+ is_int,
33
42
  replace_pipes_in_dict,
34
43
  )
35
- from meerschaum.utils.typing import List, Dict, Any, Union
36
44
  import meerschaum.core.User
37
- import datetime
38
- pipes_endpoint = endpoints['pipes']
39
- from fastapi.responses import StreamingResponse
40
- import io
45
+
46
+ fastapi_responses = attempt_import('fastapi.responses', lazy=False)
47
+ StreamingResponse = fastapi_responses.StreamingResponse
41
48
  dateutil_parser = attempt_import('dateutil.parser', lazy=False)
42
- pd = attempt_import('pandas')
49
+ pipes_endpoint = endpoints['pipes']
50
+ pd = attempt_import('pandas', lazy=False)
43
51
 
44
52
 
45
53
  @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
@@ -79,13 +87,13 @@ def register_pipe(
79
87
 
80
88
  @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
81
89
  def delete_pipe(
82
- connector_keys: str,
83
- metric_key: str,
84
- location_key: str,
85
- curr_user = (
86
- fastapi.Depends(manager) if not no_auth else None
87
- ),
88
- ):
90
+ connector_keys: str,
91
+ metric_key: str,
92
+ location_key: str,
93
+ curr_user = (
94
+ fastapi.Depends(manager) if not no_auth else None
95
+ ),
96
+ ):
89
97
  """
90
98
  Delete a Pipe (without dropping its table).
91
99
  """
@@ -141,15 +149,15 @@ def drop_pipe(
141
149
 
142
150
  @app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
143
151
  def edit_pipe(
144
- connector_keys: str,
145
- metric_key: str,
146
- location_key: str,
147
- parameters: dict,
148
- patch: bool = False,
149
- curr_user = (
150
- fastapi.Depends(manager) if not no_auth else None
151
- ),
152
- ):
152
+ connector_keys: str,
153
+ metric_key: str,
154
+ location_key: str,
155
+ parameters: dict,
156
+ patch: bool = False,
157
+ curr_user = (
158
+ fastapi.Depends(manager) if not no_auth else None
159
+ ),
160
+ ):
153
161
  """
154
162
  Edit an existing pipe.
155
163
  """
@@ -177,15 +185,15 @@ def edit_pipe(
177
185
 
178
186
  @app.get(pipes_endpoint + '/keys', tags=['Pipes'])
179
187
  async def fetch_pipes_keys(
180
- connector_keys: str = "[]",
181
- metric_keys: str = "[]",
182
- location_keys: str = "[]",
183
- tags: str = "[]",
184
- params: str = "{}",
185
- curr_user = (
186
- fastapi.Depends(manager) if not no_auth else None
187
- ),
188
- ):
188
+ connector_keys: str = "[]",
189
+ metric_keys: str = "[]",
190
+ location_keys: str = "[]",
191
+ tags: str = "[]",
192
+ params: str = "{}",
193
+ curr_user = (
194
+ fastapi.Depends(manager) if not no_auth else None
195
+ ),
196
+ ):
189
197
  """
190
198
  Get a list of tuples of all registered Pipes' keys.
191
199
  """
@@ -201,14 +209,14 @@ async def fetch_pipes_keys(
201
209
 
202
210
  @app.get(pipes_endpoint, tags=['Pipes'])
203
211
  async def get_pipes(
204
- connector_keys: str = "",
205
- metric_keys: str = "",
206
- location_keys: str = "",
207
- curr_user=(
208
- fastapi.Depends(manager) if not no_auth else None
209
- ),
210
- debug: bool = False,
211
- ) -> Dict[str, Any]:
212
+ connector_keys: str = "",
213
+ metric_keys: str = "",
214
+ location_keys: str = "",
215
+ curr_user=(
216
+ fastapi.Depends(manager) if not no_auth else None
217
+ ),
218
+ debug: bool = False,
219
+ ) -> Dict[str, Any]:
212
220
  """
213
221
  Get all registered Pipes with metadata, excluding parameters.
214
222
  """
@@ -223,11 +231,11 @@ async def get_pipes(
223
231
 
224
232
  @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
225
233
  async def get_pipes_by_connector(
226
- connector_keys: str,
227
- curr_user = (
228
- fastapi.Depends(manager) if not no_auth else None
229
- ),
230
- ) -> Dict[str, Any]:
234
+ connector_keys: str,
235
+ curr_user = (
236
+ fastapi.Depends(manager) if not no_auth else None
237
+ ),
238
+ ) -> Dict[str, Any]:
231
239
  """
232
240
  Get all registered Pipes by connector_keys with metadata, excluding parameters.
233
241
  """
@@ -240,13 +248,13 @@ async def get_pipes_by_connector(
240
248
 
241
249
  @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
242
250
  async def get_pipes_by_connector_and_metric(
243
- connector_keys: str,
244
- metric_key: str,
245
- parent: bool = False,
246
- curr_user = (
247
- fastapi.Depends(manager) if not no_auth else None
248
- ),
249
- ):
251
+ connector_keys: str,
252
+ metric_key: str,
253
+ parent: bool = False,
254
+ curr_user = (
255
+ fastapi.Depends(manager) if not no_auth else None
256
+ ),
257
+ ):
250
258
  """
251
259
  Get all registered Pipes by connector_keys and metric_key with metadata, excluding parameters.
252
260
 
@@ -269,13 +277,13 @@ async def get_pipes_by_connector_and_metric(
269
277
 
270
278
  @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
271
279
  async def get_pipes_by_connector_and_metric_and_location(
272
- connector_keys: str,
273
- metric_key: str,
274
- location_key: str,
275
- curr_user = (
276
- fastapi.Depends(manager) if not no_auth else None
277
- ),
278
- ):
280
+ connector_keys: str,
281
+ metric_key: str,
282
+ location_key: str,
283
+ curr_user = (
284
+ fastapi.Depends(manager) if not no_auth else None
285
+ ),
286
+ ):
279
287
  """
280
288
  Get a specific Pipe with metadata, excluding parameters.
281
289
  """
@@ -297,17 +305,17 @@ async def get_pipes_by_connector_and_metric_and_location(
297
305
 
298
306
  @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
299
307
  def get_sync_time(
300
- connector_keys: str,
301
- metric_key: str,
302
- location_key: str,
303
- params: Optional[Dict[str, Any]] = None,
304
- newest: bool = True,
305
- round_down: bool = True,
306
- debug: bool = False,
307
- curr_user = (
308
- fastapi.Depends(manager) if not no_auth else None
309
- ),
310
- ) -> Union[str, int, None]:
308
+ connector_keys: str,
309
+ metric_key: str,
310
+ location_key: str,
311
+ params: Optional[Dict[str, Any]] = None,
312
+ newest: bool = True,
313
+ round_down: bool = True,
314
+ debug: bool = False,
315
+ curr_user = (
316
+ fastapi.Depends(manager) if not no_auth else None
317
+ ),
318
+ ) -> Union[str, int, None]:
311
319
  """
312
320
  Get a Pipe's latest datetime value.
313
321
  See `meerschaum.Pipe.get_sync_time`.
@@ -328,20 +336,20 @@ def get_sync_time(
328
336
 
329
337
  @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
330
338
  def sync_pipe(
331
- connector_keys: str,
332
- metric_key: str,
333
- location_key: str,
334
- data: dict = None,
335
- check_existing: bool = True,
336
- blocking: bool = True,
337
- force: bool = False,
338
- workers: Optional[int] = None,
339
- columns: Optional[str] = None,
340
- curr_user = (
341
- fastapi.Depends(manager) if not no_auth else None
342
- ),
343
- debug: bool = False,
344
- ) -> List[Union[bool, str]]:
339
+ connector_keys: str,
340
+ metric_key: str,
341
+ location_key: str,
342
+ data: dict = None,
343
+ check_existing: bool = True,
344
+ blocking: bool = True,
345
+ force: bool = False,
346
+ workers: Optional[int] = None,
347
+ columns: Optional[str] = None,
348
+ curr_user = (
349
+ fastapi.Depends(manager) if not no_auth else None
350
+ ),
351
+ debug: bool = False,
352
+ ) -> List[Union[bool, str]]:
345
353
  """
346
354
  Add data to an existing Pipe.
347
355
  See `meerschaum.Pipe.sync`.
@@ -365,11 +373,11 @@ def sync_pipe(
365
373
 
366
374
  result = list(p.sync(
367
375
  data,
368
- debug = debug,
369
- check_existing = check_existing,
370
- blocking = blocking,
371
- force = force,
372
- workers = workers
376
+ debug=debug,
377
+ check_existing=check_existing,
378
+ blocking=blocking,
379
+ force=force,
380
+ workers=workers,
373
381
  ))
374
382
  return result
375
383
 
@@ -581,7 +589,7 @@ def get_pipe_attributes(
581
589
  ),
582
590
  ) -> Dict[str, Any]:
583
591
  """Get a Pipe's attributes."""
584
- return get_pipe(connector_keys, metric_key, location_key).attributes
592
+ return get_pipe(connector_keys, metric_key, location_key, refresh=True).attributes
585
593
 
586
594
 
587
595
  @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.6.0"
5
+ __version__ = "2.6.1"
@@ -367,6 +367,7 @@ def get_pipe_data(
367
367
 
368
368
  from meerschaum.utils.packages import import_pandas
369
369
  from meerschaum.utils.dataframe import parse_df_datetimes, add_missing_cols_to_df
370
+ from meerschaum.utils.dtypes import are_dtypes_equal
370
371
  pd = import_pandas()
371
372
  try:
372
373
  df = pd.read_json(StringIO(response.text))
@@ -382,16 +383,17 @@ def get_pipe_data(
382
383
  ignore_cols = [
383
384
  col
384
385
  for col, dtype in pipe.dtypes.items()
385
- if 'datetime' not in str(dtype)
386
+ if not are_dtypes_equal(str(dtype), 'datetime')
386
387
  ],
387
- debug = debug,
388
+ strip_timezone=(pipe.tzinfo is None),
389
+ debug=debug,
388
390
  )
389
391
  return df
390
392
 
391
393
 
392
394
  def get_pipe_id(
393
395
  self,
394
- pipe: meerschuam.Pipe,
396
+ pipe: mrsm.Pipe,
395
397
  debug: bool = False,
396
398
  ) -> int:
397
399
  """Get a Pipe's ID from the API."""
@@ -413,7 +415,7 @@ def get_pipe_id(
413
415
 
414
416
  def get_pipe_attributes(
415
417
  self,
416
- pipe: meerschaum.Pipe,
418
+ pipe: mrsm.Pipe,
417
419
  debug: bool = False,
418
420
  ) -> Dict[str, Any]:
419
421
  """Get a Pipe's attributes from the API
@@ -439,7 +441,7 @@ def get_pipe_attributes(
439
441
 
440
442
  def get_sync_time(
441
443
  self,
442
- pipe: 'meerschaum.Pipe',
444
+ pipe: mrsm.Pipe,
443
445
  params: Optional[Dict[str, Any]] = None,
444
446
  newest: bool = True,
445
447
  debug: bool = False,
@@ -90,16 +90,18 @@ def fetch(
90
90
  )
91
91
  ### if sqlite, parse for datetimes
92
92
  if not as_hook_results and self.flavor == 'sqlite':
93
- from meerschaum.utils.misc import parse_df_datetimes
93
+ from meerschaum.utils.dataframe import parse_df_datetimes
94
+ from meerschaum.utils.dtypes import are_dtypes_equal
94
95
  ignore_cols = [
95
96
  col
96
97
  for col, dtype in pipe.dtypes.items()
97
- if 'datetime' not in str(dtype)
98
+ if not are_dtypes_equal(str(dtype), 'datetime')
98
99
  ]
99
100
  return (
100
101
  parse_df_datetimes(
101
102
  chunk,
102
103
  ignore_cols=ignore_cols,
104
+ strip_timezone=(pipe.tzinfo is None),
103
105
  debug=debug,
104
106
  )
105
107
  for chunk in chunks
@@ -872,7 +872,11 @@ def get_pipe_data(
872
872
  from meerschaum.utils.sql import sql_item_name
873
873
  from meerschaum.utils.misc import parse_df_datetimes, to_pandas_dtype
874
874
  from meerschaum.utils.packages import import_pandas
875
- from meerschaum.utils.dtypes import attempt_cast_to_numeric, attempt_cast_to_uuid
875
+ from meerschaum.utils.dtypes import (
876
+ attempt_cast_to_numeric,
877
+ attempt_cast_to_uuid,
878
+ are_dtypes_equal,
879
+ )
876
880
  pd = import_pandas()
877
881
  is_dask = 'dask' in pd.__name__
878
882
 
@@ -967,7 +971,7 @@ def get_pipe_data(
967
971
  ignore_dt_cols = [
968
972
  col
969
973
  for col, dtype in pipe.dtypes.items()
970
- if 'datetime' not in str(dtype)
974
+ if not are_dtypes_equal(str(dtype), 'datetime')
971
975
  ]
972
976
  ### NOTE: We have to consume the iterator here to ensure that datetimes are parsed correctly
973
977
  df = (
@@ -975,6 +979,7 @@ def get_pipe_data(
975
979
  df,
976
980
  ignore_cols=ignore_dt_cols,
977
981
  chunksize=kw.get('chunksize', None),
982
+ strip_timezone=(pipe.tzinfo is None),
978
983
  debug=debug,
979
984
  ) if isinstance(df, pd.DataFrame) else (
980
985
  [
@@ -982,6 +987,7 @@ def get_pipe_data(
982
987
  c,
983
988
  ignore_cols=ignore_dt_cols,
984
989
  chunksize=kw.get('chunksize', None),
990
+ strip_timezone=(pipe.tzinfo is None),
985
991
  debug=debug,
986
992
  )
987
993
  for c in df
@@ -1090,6 +1096,8 @@ def get_pipe_data_query(
1090
1096
  if begin is not None:
1091
1097
  begin -= backtrack_interval
1092
1098
 
1099
+ begin, end = pipe.parse_date_bounds(begin, end)
1100
+
1093
1101
  cols_names = [
1094
1102
  sql_item_name(col, self.flavor, None)
1095
1103
  for col in select_columns
@@ -1661,11 +1669,14 @@ def sync_pipe(
1661
1669
  'autoincrement': False,
1662
1670
  },
1663
1671
  )
1664
- temp_pipe._columns_types = {
1665
- col: get_db_type_from_pd_type(str(typ), self.flavor)
1672
+ temp_pipe.__dict__['_columns_types'] = {
1673
+ col: get_db_type_from_pd_type(
1674
+ pipe.dtypes.get(col, str(typ)),
1675
+ self.flavor,
1676
+ )
1666
1677
  for col, typ in update_df.dtypes.items()
1667
1678
  }
1668
- temp_pipe._columns_types_timestamp = time.perf_counter()
1679
+ temp_pipe.__dict__['_columns_types_timestamp'] = time.perf_counter()
1669
1680
  temp_success, temp_msg = temp_pipe.sync(update_df, check_existing=False, debug=debug)
1670
1681
  if not temp_success:
1671
1682
  return temp_success, temp_msg
@@ -409,6 +409,7 @@ def get_pipe_data(
409
409
  return None
410
410
 
411
411
  from meerschaum.utils.dataframe import query_df, parse_df_datetimes
412
+ from meerschaum.utils.dtypes import are_dtypes_equal
412
413
 
413
414
  valkey_dtypes = pipe.parameters.get('valkey', {}).get('dtypes', {})
414
415
  dt_col = pipe.columns.get('datetime', None)
@@ -442,13 +443,14 @@ def get_pipe_data(
442
443
  ignore_dt_cols = [
443
444
  col
444
445
  for col, dtype in pipe.dtypes.items()
445
- if 'datetime' not in str(dtype)
446
+ if not are_dtypes_equal(str(dtype), 'datetime')
446
447
  ]
447
448
 
448
449
  df = parse_df_datetimes(
449
450
  docs,
450
451
  ignore_cols=ignore_dt_cols,
451
452
  chunksize=kwargs.get('chunksize', None),
453
+ strip_timezone=(pipe.tzinfo is None),
452
454
  debug=debug,
453
455
  )
454
456
  for col, typ in valkey_dtypes.items():
@@ -105,6 +105,7 @@ class Pipe:
105
105
  autoincrement,
106
106
  upsert,
107
107
  static,
108
+ tzinfo,
108
109
  get_columns,
109
110
  get_columns_types,
110
111
  get_columns_indices,
@@ -8,10 +8,13 @@ Fetch and manipulate Pipes' attributes
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ from datetime import timezone
12
+
11
13
  import meerschaum as mrsm
12
14
  from meerschaum.utils.typing import Tuple, Dict, SuccessTuple, Any, Union, Optional, List
13
15
  from meerschaum.utils.warnings import warn
14
16
 
17
+
15
18
  @property
16
19
  def attributes(self) -> Dict[str, Any]:
17
20
  """
@@ -52,6 +55,13 @@ def parameters(self) -> Optional[Dict[str, Any]]:
52
55
  """
53
56
  if 'parameters' not in self.attributes:
54
57
  self.attributes['parameters'] = {}
58
+ _parameters = self.attributes['parameters']
59
+ dt_col = _parameters.get('columns', {}).get('datetime', None)
60
+ dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
61
+ if dt_col and not dt_typ:
62
+ if 'dtypes' not in _parameters:
63
+ self.attributes['parameters']['dtypes'] = {}
64
+ self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
55
65
  return self.attributes['parameters']
56
66
 
57
67
 
@@ -260,6 +270,25 @@ def autoincrement(self, _autoincrement: bool) -> None:
260
270
  self.parameters['autoincrement'] = _autoincrement
261
271
 
262
272
 
273
+ @property
274
+ def tzinfo(self) -> Union[None, timezone]:
275
+ """
276
+ Return `timezone.utc` if the pipe is timezone-aware.
277
+ """
278
+ dt_col = self.columns.get('datetime', None)
279
+ if not dt_col:
280
+ return None
281
+
282
+ dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
283
+ if 'utc' in dt_typ.lower() or dt_typ == 'datetime':
284
+ return timezone.utc
285
+
286
+ if dt_typ == 'datetime64[ns]':
287
+ return None
288
+
289
+ return None
290
+
291
+
263
292
  def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
264
293
  """
265
294
  Check if the requested columns are defined.
@@ -30,6 +30,7 @@ def enforce_dtypes(
30
30
  from meerschaum.utils.warnings import warn
31
31
  from meerschaum.utils.debug import dprint
32
32
  from meerschaum.utils.dataframe import parse_df_datetimes, enforce_dtypes as _enforce_dtypes
33
+ from meerschaum.utils.dtypes import are_dtypes_equal
33
34
  from meerschaum.utils.packages import import_pandas
34
35
  pd = import_pandas(debug=debug)
35
36
  if df is None:
@@ -51,6 +52,7 @@ def enforce_dtypes(
51
52
  for col, dtype in pipe_dtypes.items()
52
53
  if 'datetime' not in str(dtype)
53
54
  ],
55
+ strip_timezone=(self.tzinfo is None),
54
56
  chunksize=chunksize,
55
57
  debug=debug,
56
58
  )
@@ -60,8 +62,9 @@ def enforce_dtypes(
60
62
  ignore_cols=[
61
63
  col
62
64
  for col, dtype in pipe_dtypes.items()
63
- if 'datetime' not in str(dtype)
65
+ if not are_dtypes_equal(str(dtype), 'datetime')
64
66
  ],
67
+ strip_timezone=(self.tzinfo is None),
65
68
  chunksize=chunksize,
66
69
  debug=debug,
67
70
  )
@@ -77,7 +80,14 @@ def enforce_dtypes(
77
80
  )
78
81
  return df
79
82
 
80
- return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
83
+ return _enforce_dtypes(
84
+ df,
85
+ pipe_dtypes,
86
+ safe_copy=safe_copy,
87
+ strip_timezone=(self.tzinfo is None),
88
+ coerce_timezone=True,
89
+ debug=debug,
90
+ )
81
91
 
82
92
 
83
93
  def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
@@ -933,7 +933,11 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
933
933
  if not new_numeric_cols:
934
934
  return True, "Success"
935
935
 
936
+ self._attributes_sync_time = None
937
+ dt_col = self.columns.get('datetime', None)
936
938
  dtypes = self.parameters.get('dtypes', {})
939
+ if dt_col not in dtypes:
940
+ dtypes[dt_col] = 'datetime'
937
941
  dtypes.update({col: 'numeric' for col in numeric_cols})
938
942
  self.parameters['dtypes'] = dtypes
939
943
  if not self.temporary:
@@ -957,7 +961,11 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
957
961
  if not new_uuid_cols:
958
962
  return True, "Success"
959
963
 
964
+ self._attributes_sync_time = None
965
+ dt_col = self.columns.get('datetime', None)
960
966
  dtypes = self.parameters.get('dtypes', {})
967
+ if dt_col not in dtypes:
968
+ dtypes[dt_col] = 'datetime'
961
969
  dtypes.update({col: 'uuid' for col in uuid_cols})
962
970
  self.parameters['dtypes'] = dtypes
963
971
  if not self.temporary:
@@ -981,7 +989,11 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
981
989
  if not new_json_cols:
982
990
  return True, "Success"
983
991
 
992
+ self._attributes_sync_time = None
993
+ dt_col = self.columns.get('datetime', None)
984
994
  dtypes = self.parameters.get('dtypes', {})
995
+ if dt_col not in dtypes:
996
+ dtypes[dt_col] = 'datetime'
985
997
  dtypes.update({col: 'json' for col in json_cols})
986
998
  self.parameters['dtypes'] = dtypes
987
999
 
@@ -689,6 +689,7 @@ def enforce_dtypes(
689
689
  safe_copy: bool = True,
690
690
  coerce_numeric: bool = True,
691
691
  coerce_timezone: bool = True,
692
+ strip_timezone: bool = False,
692
693
  debug: bool = False,
693
694
  ) -> 'pd.DataFrame':
694
695
  """
@@ -713,6 +714,10 @@ def enforce_dtypes(
713
714
  coerce_timezone: bool, default True
714
715
  If `True`, convert datetimes to UTC.
715
716
 
717
+ strip_timezone: bool, default False
718
+ If `coerce_timezone` and `strip_timezone` are `True`,
719
+ remove timezone information from datetimes.
720
+
716
721
  debug: bool, default False
717
722
  Verbosity toggle.
718
723
 
@@ -817,7 +822,7 @@ def enforce_dtypes(
817
822
  dprint(f"Checking for datetime conversion: {datetime_cols}")
818
823
  for col in datetime_cols:
819
824
  if col in df.columns:
820
- df[col] = _coerce_timezone(df[col])
825
+ df[col] = _coerce_timezone(df[col], strip_utc=strip_timezone)
821
826
 
822
827
  df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
823
828
  if are_dtypes_equal(df_dtypes, pipe_pandas_dtypes):
@@ -304,6 +304,19 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
304
304
  'cockroachdb': 'Float',
305
305
  'default': 'Float',
306
306
  },
307
+ 'datetime': {
308
+ 'timescaledb': 'DateTime(timezone=True)',
309
+ 'postgresql': 'DateTime(timezone=True)',
310
+ 'mariadb': 'DateTime(timezone=True)',
311
+ 'mysql': 'DateTime(timezone=True)',
312
+ 'mssql': 'sqlalchemy.dialects.mssql.DATETIMEOFFSET',
313
+ 'oracle': 'sqlalchemy.dialects.oracle.TIMESTAMP(timezone=True)',
314
+ 'sqlite': 'DateTime(timezone=True)',
315
+ 'duckdb': 'DateTime(timezone=True)',
316
+ 'citus': 'DateTime(timezone=True)',
317
+ 'cockroachdb': 'DateTime(timezone=True)',
318
+ 'default': 'DateTime(timezone=True)',
319
+ },
307
320
  'datetime64[ns]': {
308
321
  'timescaledb': 'DateTime',
309
322
  'postgresql': 'DateTime',
@@ -489,7 +502,7 @@ def get_db_type_from_pd_type(
489
502
  """
490
503
  from meerschaum.utils.warnings import warn
491
504
  from meerschaum.utils.packages import attempt_import
492
- from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_PD_DTYPES
505
+ from meerschaum.utils.dtypes import are_dtypes_equal
493
506
  from meerschaum.utils.misc import parse_arguments_str
494
507
  sqlalchemy_types = attempt_import('sqlalchemy.types')
495
508
 
meerschaum/utils/sql.py CHANGED
@@ -606,7 +606,7 @@ def dateadd_str(
606
606
  da = begin + (f" + INTERVAL '{number} {datepart}'" if number != 0 else '')
607
607
 
608
608
  elif flavor in ('mssql',):
609
- if begin_time and begin_time.microsecond != 0:
609
+ if begin_time and begin_time.microsecond != 0 and not dt_is_utc:
610
610
  begin = begin[:-4] + "'"
611
611
  begin = f"CAST({begin} AS {db_type})" if begin != 'now' else 'GETUTCDATE()'
612
612
  da = f"DATEADD({datepart}, {number}, {begin})" if number != 0 else begin
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: meerschaum
3
- Version: 2.6.0
3
+ Version: 2.6.1
4
4
  Summary: Sync Time-Series Pipes with Meerschaum
5
5
  Home-page: https://meerschaum.io
6
6
  Author: Bennett Meares
@@ -124,7 +124,7 @@ meerschaum/api/routes/_index.py,sha256=QI6CBo6pI2Zi0a6fJHDjZfiLa9f4okb0BGe3A_JD0
124
124
  meerschaum/api/routes/_jobs.py,sha256=deKv9Y9XtCAB6u3ZyI2_zeMbrzkA8Q8-o6PWmTf9sxc,11702
125
125
  meerschaum/api/routes/_login.py,sha256=ti2onNOemOGLHLoPubAQOYtD7eq7FA1jOlbOSVSjXVo,2466
126
126
  meerschaum/api/routes/_misc.py,sha256=05--9ZVFeaCgZrHER2kA3SYdK4TyfkEXOCjLvPbum-w,2469
127
- meerschaum/api/routes/_pipes.py,sha256=BXk3jHmpI31XVRt9ffe6DVhoF-44NvIHgB6-b2EVAos,21753
127
+ meerschaum/api/routes/_pipes.py,sha256=li52WQUwZN8JEPyIuWN7s4wASWNV2BTVAUpQ_E1CKLg,21579
128
128
  meerschaum/api/routes/_plugins.py,sha256=vR6-uTJraY1YEJMuRvds1-xFLB2mexxnp2dJwN_0rVo,6216
129
129
  meerschaum/api/routes/_users.py,sha256=SfAkZFKrKnGjpzj8SFIKzPemzQJOH3oB72h19EaUvcQ,7204
130
130
  meerschaum/api/routes/_version.py,sha256=2t-nw_9IxCVZCNEar0LOwmut2zsClRVHjiOOUx16cu0,825
@@ -143,7 +143,7 @@ meerschaum/config/_preprocess.py,sha256=-AEA8m_--KivZwTQ1sWN6LTn5sio_fUr2XZ51BO6
143
143
  meerschaum/config/_read_config.py,sha256=RLC3HHi_1ndj7ITVDKLD9_uULY3caGRwSz3ATYE-ixA,15014
144
144
  meerschaum/config/_shell.py,sha256=46_m49Txc5q1rGfCgO49ca48BODx45DQJi8D0zz1R18,4245
145
145
  meerschaum/config/_sync.py,sha256=jHcWRkxd82_BgX8Xo8agsWvf7BSbv3qHLWmYl6ehp_0,4242
146
- meerschaum/config/_version.py,sha256=mDp2JNbbYtPfewvugU_clzJtHOCwT7pYadNMMCccEkc,71
146
+ meerschaum/config/_version.py,sha256=UzTvltItc-FIu6-14JoVccascBesomfTc98NkRsgSy4,71
147
147
  meerschaum/config/paths.py,sha256=JjibeGN3YAdSNceRwsd42aNmeUrIgM6ndzC8qZAmNI0,621
148
148
  meerschaum/config/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
149
149
  meerschaum/config/stack/__init__.py,sha256=2UukC0Lmk-aVL1o1qXzumqmuIrw3vu9fD7iCuz4XD4I,10544
@@ -163,7 +163,7 @@ meerschaum/connectors/api/_fetch.py,sha256=Khq9AFr1nk8Dsmcedb77aWhAuHw0JGgVeahDG
163
163
  meerschaum/connectors/api/_jobs.py,sha256=N5lpHFGG10jlVgaJeWAOTuLBQw3AdgjXsEPpp1YwZQE,11270
164
164
  meerschaum/connectors/api/_login.py,sha256=5GsD-B214vr5EYfM3XrTUs1sTFApxZA-9dNxq8oNSyg,2050
165
165
  meerschaum/connectors/api/_misc.py,sha256=OZRZBYOokKIEjmQaR8jUYgu6ZRn9VzXBChzR8CfDv_w,1092
166
- meerschaum/connectors/api/_pipes.py,sha256=jNqG8RYohWoijgfktfDUYExEisG8VCE_k3IMg-1FMWg,21211
166
+ meerschaum/connectors/api/_pipes.py,sha256=wHPfIqiJyU3iEQvEW61cFlriQVBvgx6hTL7VjLRgv2Q,21308
167
167
  meerschaum/connectors/api/_plugins.py,sha256=z04tPjfZZWwa7T60mogZH3X3wDmeLdnoN5Oh8m_YsU8,5217
168
168
  meerschaum/connectors/api/_request.py,sha256=Wc4Y70t0VxEj3_ch5fAeeoSAeFMuRnyNLOV-aUFInjY,6996
169
169
  meerschaum/connectors/api/_uri.py,sha256=HWxqGx4R1cHZ3ywy9Ro9ePbFxxusw4RLaC3hpGt9Z6I,1469
@@ -174,9 +174,9 @@ meerschaum/connectors/sql/_SQLConnector.py,sha256=PQHXbYdcKx7LpiE7-pyEhCeS55W3oI
174
174
  meerschaum/connectors/sql/__init__.py,sha256=3cqYiDkVasn7zWdtOTAZbT4bo95AuvGOmDD2TkaAxtw,205
175
175
  meerschaum/connectors/sql/_cli.py,sha256=1SgnWeMIAihoxp4FzbNrcq1npXf0dSOQnCntpU9hUXA,4405
176
176
  meerschaum/connectors/sql/_create_engine.py,sha256=zqeu1xHOw3n3Zgfjx-diy2aoynfdOlfOjwFuRrzB028,10452
177
- meerschaum/connectors/sql/_fetch.py,sha256=J5e9EZrb70neXq8p78LFTTJiVukMIEKHuTXlILHDDGI,13005
177
+ meerschaum/connectors/sql/_fetch.py,sha256=A2R1aLgdEkiIet8P2BZ13OXy31I5e8BzWee74Bj_v4k,13141
178
178
  meerschaum/connectors/sql/_instance.py,sha256=3KJI3ImwWAJkUfdZIrSL24pcW6Nic8wo5IUeGth9HP4,6459
179
- meerschaum/connectors/sql/_pipes.py,sha256=AjSEDE6SPGifVUQp1GNmk8y3HUgD6wESOrEDsHk-1-4,115768
179
+ meerschaum/connectors/sql/_pipes.py,sha256=GOPMjp5HUHp4hQa8Yxm4GAodIGW1FlSGm8uUUkUKDPA,116097
180
180
  meerschaum/connectors/sql/_plugins.py,sha256=wbxcNxqTtjfDsxPvdVGTllasYf6NHHzODaQ72hEUSBQ,8135
181
181
  meerschaum/connectors/sql/_sql.py,sha256=qPJbm0dAC3oY4ZrCFnkFCI74OqwFBWlhJnryr9faao4,37597
182
182
  meerschaum/connectors/sql/_uri.py,sha256=0BrhQtqQdzg9mR04gWBZINs_BbPFtSlTECXT_TCUwik,3460
@@ -187,12 +187,12 @@ meerschaum/connectors/sql/tables/types.py,sha256=Jc_MTHIBM-KHpQt__Lckp39CeOo7tGO
187
187
  meerschaum/connectors/valkey/_ValkeyConnector.py,sha256=xHld4OCnt0SXmAWH8Yintb3931F-MEgXfWerhTthsXc,15849
188
188
  meerschaum/connectors/valkey/__init__.py,sha256=jkVutsygQCvGPLN17cP6wHAjHajxVycnQJbm2eVMuY0,187
189
189
  meerschaum/connectors/valkey/_fetch.py,sha256=MjeE0h3YI4M3LCzy7axQAc_fX_l82vUqX4WXcYoppxE,1920
190
- meerschaum/connectors/valkey/_pipes.py,sha256=DP5dGWZf6PbJLkCI5HjPMq44KcEj9j_K0Z0yPR4CjaU,22723
190
+ meerschaum/connectors/valkey/_pipes.py,sha256=9sdKl605cNuE6rx4hlwgKEiIlU4fomtrz5nG_z8yxEU,22842
191
191
  meerschaum/connectors/valkey/_plugins.py,sha256=ZqiEW4XZCOpw4G8DUK2IKY6Qrph4mYfTjgXWimgakYY,6267
192
192
  meerschaum/connectors/valkey/_users.py,sha256=AS1vLarrkDA9yPK644GWwRiQiTZVa9x3nlLpyntq40g,7730
193
193
  meerschaum/core/__init__.py,sha256=tjASW10n9uLV6bYhcwP4rggh-ESXSJzgxpSBbVsuISs,251
194
- meerschaum/core/Pipe/__init__.py,sha256=9Pe9mCzxM9kJdt005tq6jfTYijfVyaoHjA8k5MmGCCk,18513
195
- meerschaum/core/Pipe/_attributes.py,sha256=w-zHzsRRtdYQxqdCHncQqp5ZmULnwdyBZHNTw75vRq0,20438
194
+ meerschaum/core/Pipe/__init__.py,sha256=knkz0gVpIrHb8au2b_YxvnSC9eXMgPKr5b7TxVQE9O0,18529
195
+ meerschaum/core/Pipe/_attributes.py,sha256=7zW4ppZnQPkRoSpRB7hX2WJuumXe-7g9pCvSM-7HeGA,21299
196
196
  meerschaum/core/Pipe/_bootstrap.py,sha256=evyi07kkzAVMj66HfZkbYdcWk_oHUDsl6f13EnSPMYs,7723
197
197
  meerschaum/core/Pipe/_clear.py,sha256=LghXabgyyc1tD7FNQrh9ExT71ipcg2poM9FDA3k9e4M,2230
198
198
  meerschaum/core/Pipe/_copy.py,sha256=YDclAapf_spm9phpFr4-CALyYyw7nUsyKyiaLM1cnm4,2965
@@ -200,12 +200,12 @@ meerschaum/core/Pipe/_data.py,sha256=HIxt5JiuOba7vzz03P3wdAgbKJ3j-AUPRwdvu14e8b8
200
200
  meerschaum/core/Pipe/_deduplicate.py,sha256=xthUdsDxGO2t3m0XGDm9K3F6dpaZoemtjNi8gyKm0e0,10177
201
201
  meerschaum/core/Pipe/_delete.py,sha256=1geNp9BgrocXP1gt76dMbnlJWKYFMuSNqPFA4K4-hXE,2118
202
202
  meerschaum/core/Pipe/_drop.py,sha256=gIKdHWm0RsIgC2nu768MtFnPbCXyiJjNxaiT1obyuIo,1036
203
- meerschaum/core/Pipe/_dtypes.py,sha256=8cp1TwmhXjLvQ1OpgKNdpIP9af0OxkfqiphXNuCNvRQ,3818
203
+ meerschaum/core/Pipe/_dtypes.py,sha256=VohahiZk44Uw3PNsTLyqH8m9wRdB-mkPgYvgCb5hlA0,4114
204
204
  meerschaum/core/Pipe/_edit.py,sha256=HrKWe9vhqKaNOjOcJzW5BNbaUBPIbgNAhJEK8OMsy7c,8416
205
205
  meerschaum/core/Pipe/_fetch.py,sha256=Q_LncNi1nv-YwvRPbh1QK0hf6hflL7Hn9v9lT3oQgF4,5451
206
206
  meerschaum/core/Pipe/_register.py,sha256=Sd5xaAW8H7uLTIoommcKb-6kHPRuHJLWNSbPnt2UbvA,2240
207
207
  meerschaum/core/Pipe/_show.py,sha256=nG50y8eBT9TVuKkRgAKtNDNIxysJvMNxfu__lkL1F9k,1352
208
- meerschaum/core/Pipe/_sync.py,sha256=_RwqLkyizqHpfOaKBpkFJpqbuebU4zXIW8achVjPZuI,33676
208
+ meerschaum/core/Pipe/_sync.py,sha256=yk5hCidGKVYyXQ03XKteWgOmflzK5M-84ILprlRG-Sk,34129
209
209
  meerschaum/core/Pipe/_verify.py,sha256=c3HvsZd4QPydqozaV6cDDRtwYiNz4V91b0IcnKvcimA,14158
210
210
  meerschaum/core/Plugin/__init__.py,sha256=UXg64EvJPgI1PCxkY_KM02-ZmBm4FZpLPIQR_uSJJDc,137
211
211
  meerschaum/core/User/_User.py,sha256=JZ9Y1tsjZe-cgD24m9YfZ6ZwSOKn_sHc4rbQ7KblBz8,6592
@@ -219,7 +219,7 @@ meerschaum/plugins/__init__.py,sha256=6krcqaMKyzuVqesXMqEL0XEy2SJQ4xfNt2-oI_fJ6v
219
219
  meerschaum/plugins/bootstrap.py,sha256=VwjpZAuYdqPJW0YoVgAoM_taHkdQHqP902-8T7OWWCI,11339
220
220
  meerschaum/utils/__init__.py,sha256=QrK1K9hIbPCRCM5k2nZGFqGnrqhA0Eh-iSmCU7FG6Cs,612
221
221
  meerschaum/utils/_get_pipes.py,sha256=tu4xKPoDn79Dz2kWM13cXTP4DSCkn-3G9M8KiLftopw,11073
222
- meerschaum/utils/dataframe.py,sha256=7Lh2KQci3qgCK_rxIboSyum9CfrGA5VFFv5ye9ohQbc,43020
222
+ meerschaum/utils/dataframe.py,sha256=Q4Rc796K5gaYiEJmmwLjPKqiJGsbj6qPtYRXdzCSaK0,43235
223
223
  meerschaum/utils/debug.py,sha256=GyIzJmunkoPnOcZNYVQdT4Sgd-aOb5MI2VbIgATOjIQ,3695
224
224
  meerschaum/utils/interactive.py,sha256=t-6jWozXSqL7lYGDHuwiOjTgr-UKhdcg61q_eR5mikI,3196
225
225
  meerschaum/utils/misc.py,sha256=soGmUooT216Dl15KbcUTzf8E-aC6uNM6Zvy1PiUT_Y4,47089
@@ -228,7 +228,7 @@ meerschaum/utils/pool.py,sha256=vkE42af4fjrTEJTxf6Ek3xGucm1MtEkpsSEiaVzNKHs,2655
228
228
  meerschaum/utils/process.py,sha256=9O8PPPJjY9Q5W2f39I3B3lFU6TlSiRiI3bgrzdOOyOw,7843
229
229
  meerschaum/utils/prompt.py,sha256=6J--mZJ_NcEdSX6KMjtY4fXXezyILLHP24VdxFFqOIc,18985
230
230
  meerschaum/utils/schedule.py,sha256=9BQGEzDbInLAU1aFO-FvL3wKu9XCTUpS0V_aQID6xzc,11228
231
- meerschaum/utils/sql.py,sha256=aNDKD45uYWgFxLW9y6zvBb5GpND8N9eJfI48ovDnO7M,70967
231
+ meerschaum/utils/sql.py,sha256=DYQRbeL2lFVj8pyn4MIJv7pQCvjOkMAvVV6ZSjlDulI,70985
232
232
  meerschaum/utils/threading.py,sha256=3N8JXPAnwqJiSjuQcbbJg3Rv9-CCUMJpeQRfKFR7MaA,2489
233
233
  meerschaum/utils/typing.py,sha256=U3MC347sh1umpa3Xr1k71eADyDmk4LB6TnVCpq8dVzI,2830
234
234
  meerschaum/utils/warnings.py,sha256=n-phr3BftNNgyPnvnXC_VMSjtCvjiCZ-ewmVfcROhkc,6611
@@ -240,7 +240,7 @@ meerschaum/utils/daemon/StdinFile.py,sha256=J6tyUReM8NEp3bBQAxMfe8mjJG5mWi6CzHN4
240
240
  meerschaum/utils/daemon/__init__.py,sha256=o9jWb4lRTIyny4EPt7fPXFgV_vIf1mUofsTwoE1ZecA,8751
241
241
  meerschaum/utils/daemon/_names.py,sha256=d2ZwTxBoTAqXZkCfZ5LuX2XrkQmLNUq1OTlUqfoH5dA,4515
242
242
  meerschaum/utils/dtypes/__init__.py,sha256=xK7duStyLFWaeeaPpz-xBAnOlC0TZ40_1Ja9Q7Mz-sM,7944
243
- meerschaum/utils/dtypes/sql.py,sha256=2sa03vMk_M8tyvZG1eWWmjFwWyhl09AJ1N70jkZ-y1E,18139
243
+ meerschaum/utils/dtypes/sql.py,sha256=IQihwQy4OKSbRjvJy6ky6SszFKR7W1iMs-ruZDsf2js,18701
244
244
  meerschaum/utils/formatting/__init__.py,sha256=GpJQWeqkdWw5IuDmW4Rgmapjzv-KkI4jhBZllJi4QIg,15999
245
245
  meerschaum/utils/formatting/_jobs.py,sha256=izsqPJhTtUkXUUtWnbXtReYsUYwulXtci3pBj72Ne64,6637
246
246
  meerschaum/utils/formatting/_pipes.py,sha256=840O5rg2aHhQoraCDOh2ZtBo43_W2W6R60yYufEoXp8,19494
@@ -251,11 +251,11 @@ meerschaum/utils/packages/_packages.py,sha256=KqQqa5ipPMuzEwy0IO3sWXbw7fLk552fZ-
251
251
  meerschaum/utils/packages/lazy_loader.py,sha256=VHnph3VozH29R4JnSSBfwtA5WKZYZQFT_GeQSShCnuc,2540
252
252
  meerschaum/utils/venv/_Venv.py,sha256=sBnlmxHdAh2bx8btfVoD79-H9-cYsv5lP02IIXkyECs,3553
253
253
  meerschaum/utils/venv/__init__.py,sha256=f3oi67lXYPLKJrnRW9lae7M3A8SFiC7DzaMoBdCVUFs,24609
254
- meerschaum-2.6.0.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
255
- meerschaum-2.6.0.dist-info/METADATA,sha256=3oGg2ZL-ilfRjUa0l_yQgrXw1BOtjO2oRbjMLjWIkMI,24704
256
- meerschaum-2.6.0.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
257
- meerschaum-2.6.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
258
- meerschaum-2.6.0.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
259
- meerschaum-2.6.0.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
260
- meerschaum-2.6.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
261
- meerschaum-2.6.0.dist-info/RECORD,,
254
+ meerschaum-2.6.1.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
255
+ meerschaum-2.6.1.dist-info/METADATA,sha256=uCifOvDOwqmmfc1DekPk4cfue5Ch_Czv3MvE8adXJG8,24704
256
+ meerschaum-2.6.1.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
257
+ meerschaum-2.6.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
258
+ meerschaum-2.6.1.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
259
+ meerschaum-2.6.1.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
260
+ meerschaum-2.6.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
261
+ meerschaum-2.6.1.dist-info/RECORD,,