meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. meerschaum/actions/copy.py +1 -0
  2. meerschaum/actions/drop.py +100 -22
  3. meerschaum/actions/index.py +71 -0
  4. meerschaum/actions/register.py +8 -12
  5. meerschaum/actions/sql.py +1 -1
  6. meerschaum/api/routes/_pipes.py +18 -0
  7. meerschaum/api/routes/_plugins.py +1 -1
  8. meerschaum/api/routes/_users.py +62 -61
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +20 -0
  11. meerschaum/connectors/sql/_SQLConnector.py +8 -12
  12. meerschaum/connectors/sql/_create_engine.py +1 -1
  13. meerschaum/connectors/sql/_fetch.py +9 -39
  14. meerschaum/connectors/sql/_instance.py +3 -3
  15. meerschaum/connectors/sql/_pipes.py +262 -70
  16. meerschaum/connectors/sql/_plugins.py +11 -16
  17. meerschaum/connectors/sql/_sql.py +60 -39
  18. meerschaum/connectors/sql/_uri.py +9 -9
  19. meerschaum/connectors/sql/_users.py +10 -12
  20. meerschaum/connectors/sql/tables/__init__.py +13 -14
  21. meerschaum/connectors/valkey/_ValkeyConnector.py +2 -2
  22. meerschaum/core/Pipe/__init__.py +12 -2
  23. meerschaum/core/Pipe/_attributes.py +32 -38
  24. meerschaum/core/Pipe/_drop.py +73 -2
  25. meerschaum/core/Pipe/_fetch.py +4 -0
  26. meerschaum/core/Pipe/_index.py +68 -0
  27. meerschaum/core/Pipe/_sync.py +16 -9
  28. meerschaum/utils/daemon/Daemon.py +9 -2
  29. meerschaum/utils/daemon/RotatingFile.py +3 -3
  30. meerschaum/utils/dataframe.py +42 -12
  31. meerschaum/utils/dtypes/__init__.py +144 -24
  32. meerschaum/utils/dtypes/sql.py +52 -9
  33. meerschaum/utils/formatting/__init__.py +2 -2
  34. meerschaum/utils/formatting/_pprint.py +12 -11
  35. meerschaum/utils/misc.py +16 -18
  36. meerschaum/utils/prompt.py +1 -1
  37. meerschaum/utils/sql.py +106 -42
  38. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/METADATA +14 -2
  39. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/RECORD +45 -43
  40. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/WHEEL +1 -1
  41. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/LICENSE +0 -0
  42. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/NOTICE +0 -0
  43. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/entry_points.txt +0 -0
  44. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/top_level.txt +0 -0
  45. {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/zip-safe +0 -0
@@ -70,6 +70,9 @@ class SQLConnector(Connector):
70
70
  create_pipe_table_from_df,
71
71
  get_pipe_columns_indices,
72
72
  get_temporary_target,
73
+ create_pipe_indices,
74
+ drop_pipe_indices,
75
+ get_pipe_index_names,
73
76
  )
74
77
  from ._plugins import (
75
78
  register_plugin,
@@ -192,7 +195,8 @@ class SQLConnector(Connector):
192
195
  self._debug = debug
193
196
  ### Store the PID and thread at initialization
194
197
  ### so we can dispose of the Pool in child processes or threads.
195
- import os, threading
198
+ import os
199
+ import threading
196
200
  self._pid = os.getpid()
197
201
  self._thread_ident = threading.current_thread().ident
198
202
  self._sessions = {}
@@ -222,7 +226,7 @@ class SQLConnector(Connector):
222
226
  return None
223
227
 
224
228
  from meerschaum.utils.packages import attempt_import
225
- sqlalchemy_orm = attempt_import('sqlalchemy.orm')
229
+ sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
226
230
  session_factory = sqlalchemy_orm.sessionmaker(self.engine)
227
231
  self._Session = sqlalchemy_orm.scoped_session(session_factory)
228
232
 
@@ -283,19 +287,17 @@ class SQLConnector(Connector):
283
287
  return ':memory:' not in self.URI
284
288
  return True
285
289
 
286
-
287
290
  @property
288
291
  def metadata(self):
289
292
  """
290
293
  Return the metadata bound to this configured schema.
291
294
  """
292
295
  from meerschaum.utils.packages import attempt_import
293
- sqlalchemy = attempt_import('sqlalchemy')
296
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
294
297
  if '_metadata' not in self.__dict__:
295
298
  self._metadata = sqlalchemy.MetaData(schema=self.schema)
296
299
  return self._metadata
297
300
 
298
-
299
301
  @property
300
302
  def instance_schema(self):
301
303
  """
@@ -303,14 +305,12 @@ class SQLConnector(Connector):
303
305
  """
304
306
  return self.schema
305
307
 
306
-
307
308
  @property
308
309
  def internal_schema(self):
309
310
  """
310
311
  Return the schema name for internal tables.
311
312
  """
312
313
  from meerschaum.config.static import STATIC_CONFIG
313
- from meerschaum.utils.packages import attempt_import
314
314
  from meerschaum.utils.sql import NO_SCHEMA_FLAVORS
315
315
  schema_name = self.__dict__.get('internal_schema', None) or (
316
316
  STATIC_CONFIG['sql']['internal_schema']
@@ -322,7 +322,6 @@ class SQLConnector(Connector):
322
322
  self._internal_schema = schema_name
323
323
  return self._internal_schema
324
324
 
325
-
326
325
  @property
327
326
  def db(self) -> Optional[databases.Database]:
328
327
  from meerschaum.utils.packages import attempt_import
@@ -339,7 +338,6 @@ class SQLConnector(Connector):
339
338
  self._db = None
340
339
  return self._db
341
340
 
342
-
343
341
  @property
344
342
  def db_version(self) -> Union[str, None]:
345
343
  """
@@ -353,7 +351,6 @@ class SQLConnector(Connector):
353
351
  self._db_version = get_db_version(self)
354
352
  return self._db_version
355
353
 
356
-
357
354
  @property
358
355
  def schema(self) -> Union[str, None]:
359
356
  """
@@ -368,12 +365,11 @@ class SQLConnector(Connector):
368
365
  self.__dict__['schema'] = None
369
366
  return None
370
367
 
371
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
368
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
372
369
  _schema = sqlalchemy.inspect(self.engine).default_schema_name
373
370
  self.__dict__['schema'] = _schema
374
371
  return _schema
375
372
 
376
-
377
373
  def __getstate__(self):
378
374
  return self.__dict__
379
375
 
@@ -186,7 +186,7 @@ def create_engine(
186
186
  """Create a sqlalchemy engine by building the engine string."""
187
187
  from meerschaum.utils.packages import attempt_import
188
188
  from meerschaum.utils.warnings import error, warn
189
- sqlalchemy = attempt_import('sqlalchemy')
189
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
190
190
  import urllib
191
191
  import copy
192
192
  ### Install and patch required drivers.
@@ -11,7 +11,7 @@ from __future__ import annotations
11
11
  from datetime import datetime, timedelta
12
12
 
13
13
  import meerschaum as mrsm
14
- from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
14
+ from meerschaum.utils.typing import Optional, Union, Any, List, Dict
15
15
 
16
16
 
17
17
  def fetch(
@@ -20,7 +20,6 @@ def fetch(
20
20
  begin: Union[datetime, int, str, None] = '',
21
21
  end: Union[datetime, int, str, None] = None,
22
22
  check_existing: bool = True,
23
- chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
24
23
  chunksize: Optional[int] = -1,
25
24
  workers: Optional[int] = None,
26
25
  debug: bool = False,
@@ -53,15 +52,12 @@ def fetch(
53
52
  check_existing: bool, defult True
54
53
  If `False`, use a backtrack interval of 0 minutes.
55
54
 
56
- chunk_hook: Callable[[pd.DataFrame], Any], default None
57
- A function to pass to `SQLConnector.read()` that accepts a Pandas DataFrame.
58
-
59
55
  chunksize: Optional[int], default -1
60
- How many rows to load into memory at once (when `chunk_hook` is provided).
56
+ How many rows to load into memory at once.
61
57
  Otherwise the entire result set is loaded into memory.
62
58
 
63
59
  workers: Optional[int], default None
64
- How many threads to use when consuming the generator (when `chunk_hook is provided).
60
+ How many threads to use when consuming the generator.
65
61
  Defaults to the number of cores.
66
62
 
67
63
  debug: bool, default False
@@ -69,8 +65,7 @@ def fetch(
69
65
 
70
66
  Returns
71
67
  -------
72
- A pandas DataFrame or `None`.
73
- If `chunk_hook` is not None, return a list of the hook function's results.
68
+ A pandas DataFrame generator.
74
69
  """
75
70
  meta_def = self.get_pipe_metadef(
76
71
  pipe,
@@ -80,33 +75,13 @@ def fetch(
80
75
  debug=debug,
81
76
  **kw
82
77
  )
83
- as_hook_results = chunk_hook is not None
84
78
  chunks = self.read(
85
79
  meta_def,
86
- chunk_hook=chunk_hook,
87
- as_hook_results=as_hook_results,
88
80
  chunksize=chunksize,
89
81
  workers=workers,
82
+ as_iterator=True,
90
83
  debug=debug,
91
84
  )
92
- ### if sqlite, parse for datetimes
93
- if not as_hook_results and self.flavor == 'sqlite':
94
- from meerschaum.utils.dataframe import parse_df_datetimes
95
- from meerschaum.utils.dtypes import are_dtypes_equal
96
- ignore_cols = [
97
- col
98
- for col, dtype in pipe.dtypes.items()
99
- if not are_dtypes_equal(str(dtype), 'datetime')
100
- ]
101
- return (
102
- parse_df_datetimes(
103
- chunk,
104
- ignore_cols=ignore_cols,
105
- strip_timezone=(pipe.tzinfo is None),
106
- debug=debug,
107
- )
108
- for chunk in chunks
109
- )
110
85
  return chunks
111
86
 
112
87
 
@@ -148,7 +123,6 @@ def get_pipe_metadef(
148
123
  from meerschaum.utils.warnings import warn
149
124
  from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
150
125
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
151
- from meerschaum.utils.misc import is_int
152
126
  from meerschaum.config import get_config
153
127
 
154
128
  dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +165,7 @@ def get_pipe_metadef(
191
165
  else begin
192
166
  )
193
167
 
194
- if begin and end and begin >= end:
168
+ if begin not in (None, '') and end is not None and begin >= end:
195
169
  begin = None
196
170
 
197
171
  if dt_name:
@@ -203,7 +177,7 @@ def get_pipe_metadef(
203
177
  begin=begin,
204
178
  db_type=db_dt_typ,
205
179
  )
206
- if begin
180
+ if begin not in ('', None)
207
181
  else None
208
182
  )
209
183
  end_da = (
@@ -214,7 +188,7 @@ def get_pipe_metadef(
214
188
  begin=end,
215
189
  db_type=db_dt_typ,
216
190
  )
217
- if end
191
+ if end is not None
218
192
  else None
219
193
  )
220
194
 
@@ -228,11 +202,7 @@ def get_pipe_metadef(
228
202
 
229
203
  has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
230
204
  if dt_name and (begin_da or end_da):
231
- definition_dt_name = (
232
- dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
233
- if not is_int((begin_da or end_da))
234
- else f"{definition_name}.{dt_name}"
235
- )
205
+ definition_dt_name = f"{definition_name}.{dt_name}"
236
206
  meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
237
207
  has_where = True
238
208
  if begin_da:
@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
25
25
  """
26
26
  from meerschaum.utils.misc import items_str
27
27
  from meerschaum.connectors.sql.tables import get_tables
28
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
28
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
29
29
  temp_tables_table = get_tables(
30
30
  mrsm_instance=self,
31
31
  create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
86
86
  """
87
87
  from meerschaum.utils.misc import items_str
88
88
  from meerschaum.connectors.sql.tables import get_tables
89
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
89
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
90
90
  temp_tables_table = get_tables(
91
91
  mrsm_instance=self,
92
92
  create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
150
150
  """
151
151
  from meerschaum.config import get_config
152
152
  from meerschaum.connectors.sql.tables import get_tables
153
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
153
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
154
154
  temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
155
155
  last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
156
156
  now_ts = time.perf_counter()