meerschaum 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. meerschaum/_internal/shell/Shell.py +4 -6
  2. meerschaum/_internal/shell/ShellCompleter.py +6 -5
  3. meerschaum/actions/clear.py +6 -3
  4. meerschaum/actions/copy.py +33 -27
  5. meerschaum/actions/drop.py +100 -22
  6. meerschaum/actions/index.py +71 -0
  7. meerschaum/actions/register.py +8 -12
  8. meerschaum/actions/sql.py +1 -1
  9. meerschaum/actions/sync.py +22 -18
  10. meerschaum/api/dash/pipes.py +2 -3
  11. meerschaum/api/routes/_pipes.py +18 -0
  12. meerschaum/api/routes/_plugins.py +1 -1
  13. meerschaum/api/routes/_users.py +62 -61
  14. meerschaum/config/_default.py +5 -0
  15. meerschaum/config/_version.py +1 -1
  16. meerschaum/connectors/api/_misc.py +3 -2
  17. meerschaum/connectors/api/_pipes.py +28 -9
  18. meerschaum/connectors/sql/_SQLConnector.py +7 -3
  19. meerschaum/connectors/sql/_create_engine.py +1 -1
  20. meerschaum/connectors/sql/_fetch.py +4 -9
  21. meerschaum/connectors/sql/_instance.py +3 -3
  22. meerschaum/connectors/sql/_pipes.py +292 -76
  23. meerschaum/connectors/sql/_plugins.py +11 -16
  24. meerschaum/connectors/sql/_sql.py +13 -9
  25. meerschaum/connectors/sql/_uri.py +9 -9
  26. meerschaum/connectors/sql/_users.py +10 -12
  27. meerschaum/connectors/sql/tables/__init__.py +13 -14
  28. meerschaum/core/Pipe/__init__.py +12 -2
  29. meerschaum/core/Pipe/_attributes.py +32 -38
  30. meerschaum/core/Pipe/_drop.py +73 -2
  31. meerschaum/core/Pipe/_index.py +68 -0
  32. meerschaum/jobs/_Job.py +1 -0
  33. meerschaum/plugins/__init__.py +7 -3
  34. meerschaum/utils/daemon/Daemon.py +5 -1
  35. meerschaum/utils/daemon/__init__.py +2 -2
  36. meerschaum/utils/dtypes/sql.py +2 -2
  37. meerschaum/utils/misc.py +7 -6
  38. meerschaum/utils/packages/__init__.py +31 -27
  39. meerschaum/utils/packages/_packages.py +1 -1
  40. meerschaum/utils/prompt.py +54 -36
  41. meerschaum/utils/sql.py +80 -34
  42. meerschaum/utils/venv/__init__.py +12 -3
  43. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/METADATA +17 -5
  44. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/RECORD +50 -48
  45. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/WHEEL +1 -1
  46. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/LICENSE +0 -0
  47. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/NOTICE +0 -0
  48. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/entry_points.txt +0 -0
  49. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/top_level.txt +0 -0
  50. {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/zip-safe +0 -0
@@ -29,7 +29,7 @@ from meerschaum.core import Plugin
29
29
  starlette_responses = attempt_import('starlette.responses', warn=False)
30
30
  FileResponse = starlette_responses.FileResponse
31
31
 
32
- sqlalchemy = attempt_import('sqlalchemy')
32
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
33
33
  plugins_endpoint = endpoints['plugins']
34
34
 
35
35
  @app.post(plugins_endpoint + '/{name}', tags=['Plugins'])
@@ -7,36 +7,34 @@ Routes for managing users
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
10
11
  from meerschaum.utils.typing import (
11
- Optional, Union, SuccessTuple, Any, Mapping, Sequence, Dict, List
12
+ Union, SuccessTuple, Any, Dict, List
12
13
  )
13
14
 
14
15
  from meerschaum.utils.packages import attempt_import
15
16
  from meerschaum.api import (
16
- fastapi, app, endpoints, get_api_connector, pipes, get_pipe,
17
- manager, debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
17
+ fastapi, app, endpoints, get_api_connector, manager,
18
+ debug, check_allow_chaining, DISALLOW_CHAINING_MESSAGE,
18
19
  no_auth, private,
19
20
  )
20
21
  from meerschaum.utils.misc import string_to_dict
21
22
  from meerschaum.config import get_config
22
- from meerschaum.api.tables import get_tables
23
- from starlette.responses import Response, JSONResponse
24
23
  from meerschaum.core import User
25
- import os, pathlib, datetime
26
24
 
27
- import meerschaum.core
28
- sqlalchemy = attempt_import('sqlalchemy')
25
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
29
26
  users_endpoint = endpoints['users']
30
27
 
31
28
  import fastapi
32
29
  from fastapi import HTTPException, Form
33
30
 
31
+
34
32
  @app.get(users_endpoint + "/me", tags=['Users'])
35
33
  def read_current_user(
36
- curr_user = (
37
- fastapi.Depends(manager) if not no_auth else None
38
- ),
39
- ) -> Dict[str, Union[str, int]]:
34
+ curr_user = (
35
+ fastapi.Depends(manager) if not no_auth else None
36
+ ),
37
+ ) -> Dict[str, Union[str, int]]:
40
38
  """
41
39
  Get information about the currently logged-in user.
42
40
  """
@@ -58,12 +56,13 @@ def read_current_user(
58
56
  ),
59
57
  }
60
58
 
59
+
61
60
  @app.get(users_endpoint, tags=['Users'])
62
61
  def get_users(
63
- curr_user = (
64
- fastapi.Depends(manager) if private else None
65
- ),
66
- ) -> List[str]:
62
+ curr_user = (
63
+ fastapi.Depends(manager) if private else None
64
+ ),
65
+ ) -> List[str]:
67
66
  """
68
67
  Get a list of the registered users.
69
68
  """
@@ -72,15 +71,15 @@ def get_users(
72
71
 
73
72
  @app.post(users_endpoint + "/register", tags=['Users'])
74
73
  def register_user(
75
- username: str = Form(None),
76
- password: str = Form(None),
77
- attributes: str = Form(None),
78
- type: str = Form(None),
79
- email: str = Form(None),
80
- curr_user = (
81
- fastapi.Depends(manager) if private else None
82
- ),
83
- ) -> SuccessTuple:
74
+ username: str = Form(None),
75
+ password: str = Form(None),
76
+ attributes: str = Form(None),
77
+ type: str = Form(None),
78
+ email: str = Form(None),
79
+ curr_user = (
80
+ fastapi.Depends(manager) if private else None
81
+ ),
82
+ ) -> SuccessTuple:
84
83
  """
85
84
  Register a new user.
86
85
  """
@@ -90,7 +89,7 @@ def register_user(
90
89
  if attributes is not None:
91
90
  try:
92
91
  attributes = string_to_dict(attributes)
93
- except Exception as e:
92
+ except Exception:
94
93
  return False, f"Invalid dictionary string received for attributes."
95
94
 
96
95
  allow_users = get_config('system', 'api', 'permissions', 'registration', 'users')
@@ -114,22 +113,22 @@ def register_user(
114
113
 
115
114
  @app.post(users_endpoint + "/edit", tags=['Users'])
116
115
  def edit_user(
117
- username: str = Form(None),
118
- password: str = Form(None),
119
- type: str = Form(None),
120
- email: str = Form(None),
121
- attributes: str = Form(None),
122
- curr_user = (
123
- fastapi.Depends(manager) if not no_auth else None
124
- ),
125
- ) -> SuccessTuple:
116
+ username: str = Form(None),
117
+ password: str = Form(None),
118
+ type: str = Form(None),
119
+ email: str = Form(None),
120
+ attributes: str = Form(None),
121
+ curr_user = (
122
+ fastapi.Depends(manager) if not no_auth else None
123
+ ),
124
+ ) -> SuccessTuple:
126
125
  """
127
126
  Edit an existing user.
128
127
  """
129
128
  if attributes is not None:
130
129
  try:
131
130
  attributes = string_to_dict(attributes)
132
- except Exception as e:
131
+ except Exception:
133
132
  return False, f"Invalid dictionary string received for attributes."
134
133
 
135
134
  user = User(username, password, email=email, attributes=attributes)
@@ -144,11 +143,11 @@ def edit_user(
144
143
 
145
144
  @app.get(users_endpoint + "/{username}/id", tags=['Users'])
146
145
  def get_user_id(
147
- username : str,
148
- curr_user = (
149
- fastapi.Depends(manager) if not no_auth else None
150
- ),
151
- ) -> Union[int, None]:
146
+ username : str,
147
+ curr_user = (
148
+ fastapi.Depends(manager) if not no_auth else None
149
+ ),
150
+ ) -> Union[int, None]:
152
151
  """
153
152
  Get a user's ID.
154
153
  """
@@ -157,23 +156,24 @@ def get_user_id(
157
156
 
158
157
  @app.get(users_endpoint + "/{username}/attributes", tags=['Users'])
159
158
  def get_user_attributes(
160
- username : str,
161
- curr_user = (
162
- fastapi.Depends(manager) if private else None
163
- ),
164
- ) -> Union[Dict[str, Any], None]:
159
+ username : str,
160
+ curr_user = (
161
+ fastapi.Depends(manager) if private else None
162
+ ),
163
+ ) -> Union[Dict[str, Any], None]:
165
164
  """
166
165
  Get a user's attributes.
167
166
  """
168
167
  return get_api_connector().get_user_attributes(User(username), debug=debug)
169
168
 
169
+
170
170
  @app.delete(users_endpoint + "/{username}", tags=['Users'])
171
171
  def delete_user(
172
- username: str,
173
- curr_user = (
174
- fastapi.Depends(manager) if not no_auth else None
175
- ),
176
- ) -> SuccessTuple:
172
+ username: str,
173
+ curr_user = (
174
+ fastapi.Depends(manager) if not no_auth else None
175
+ ),
176
+ ) -> SuccessTuple:
177
177
  """
178
178
  Delete a user.
179
179
  """
@@ -193,11 +193,11 @@ def delete_user(
193
193
 
194
194
  @app.get(users_endpoint + '/{username}/password_hash', tags=['Users'])
195
195
  def get_user_password_hash(
196
- username: str,
197
- curr_user = (
198
- fastapi.Depends(manager) if not no_auth else None
199
- ),
200
- ) -> str:
196
+ username: str,
197
+ curr_user = (
198
+ fastapi.Depends(manager) if not no_auth else None
199
+ ),
200
+ ) -> str:
201
201
  """
202
202
  If configured to allow chaining, return a user's password_hash.
203
203
  """
@@ -205,13 +205,14 @@ def get_user_password_hash(
205
205
  raise HTTPException(status_code=403, detail=DISALLOW_CHAINING_MESSAGE)
206
206
  return get_api_connector().get_user_password_hash(User(username), debug=debug)
207
207
 
208
+
208
209
  @app.get(users_endpoint + '/{username}/type', tags=['Users'])
209
210
  def get_user_type(
210
- username : str,
211
- curr_user = (
212
- fastapi.Depends(manager) if not no_auth else None
213
- ),
214
- ) -> str:
211
+ username : str,
212
+ curr_user = (
213
+ fastapi.Depends(manager) if not no_auth else None
214
+ ),
215
+ ) -> str:
215
216
  """
216
217
  If configured to allow chaining, return a user's type.
217
218
  """
@@ -76,6 +76,11 @@ default_system_config = {
76
76
  },
77
77
  'instance': {
78
78
  'stale_temporary_tables_minutes': 1440,
79
+ 'temporary_target': {
80
+ 'prefix': '_',
81
+ 'transaction_id_length': 4,
82
+ 'separator': '_',
83
+ },
79
84
  },
80
85
  'chunksize': 100_000,
81
86
  'poolclass': 'sqlalchemy.pool.QueuePool',
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.7.5"
5
+ __version__ = "2.7.7"
@@ -20,12 +20,13 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
20
20
  use_token=False,
21
21
  **kw
22
22
  ).json()
23
- except Exception as e:
23
+ except Exception:
24
24
  return None
25
25
  if isinstance(j, dict) and 'detail' in j:
26
26
  return None
27
27
  return j
28
28
 
29
+
29
30
  def get_chaining_status(self, **kw) -> Optional[bool]:
30
31
  """
31
32
  Fetch the chaining status of the API instance.
@@ -39,7 +40,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
39
40
  )
40
41
  if not response:
41
42
  return None
42
- except Exception as e:
43
+ except Exception:
43
44
  return None
44
45
 
45
46
  return response.json()
@@ -9,8 +9,7 @@ Register or fetch Pipes from the API
9
9
  from __future__ import annotations
10
10
  import time
11
11
  import json
12
- from io import StringIO
13
- from datetime import datetime
12
+ from datetime import datetime, timedelta
14
13
 
15
14
  import meerschaum as mrsm
16
15
  from meerschaum.utils.debug import dprint
@@ -178,11 +177,11 @@ def sync_pipe(
178
177
  """Sync a DataFrame into a Pipe."""
179
178
  from decimal import Decimal
180
179
  from meerschaum.utils.debug import dprint
181
- from meerschaum.utils.misc import json_serialize_datetime, items_str
180
+ from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
182
181
  from meerschaum.config import get_config
183
182
  from meerschaum.utils.packages import attempt_import
184
- from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
185
- begin = time.time()
183
+ from meerschaum.utils.dataframe import get_numeric_cols, to_json
184
+ begin = time.perf_counter()
186
185
  more_itertools = attempt_import('more_itertools')
187
186
  if df is None:
188
187
  msg = f"DataFrame is `None`. Cannot sync {pipe}."
@@ -304,9 +303,10 @@ def sync_pipe(
304
303
  num_success_chunks += 1
305
304
 
306
305
  success_tuple = True, (
307
- f"It took {round(time.time() - begin, 2)} seconds to sync {rowcount} row"
306
+ f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
307
+ + "to sync {rowcount:,} row"
308
308
  + ('s' if rowcount != 1 else '')
309
- + f" across {num_success_chunks} chunk" + ('s' if num_success_chunks != 1 else '') +
309
+ + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
310
310
  f" to {pipe}."
311
311
  )
312
312
  return success_tuple
@@ -549,10 +549,9 @@ def create_metadata(
549
549
  if debug:
550
550
  dprint("Create metadata response: {response.text}")
551
551
  try:
552
- metadata_response = json.loads(response.text)
552
+ _ = json.loads(response.text)
553
553
  except Exception as e:
554
554
  warn(f"Failed to create metadata on {self}:\n{e}")
555
- metadata_response = False
556
555
  return False
557
556
 
558
557
 
@@ -758,3 +757,23 @@ def get_pipe_columns_indices(
758
757
  warn(response.text)
759
758
  return None
760
759
  return j
760
+
761
+
762
+ def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str, str]:
763
+ """
764
+ Return the templated index names.
765
+ """
766
+ r_url = pipe_r_url(pipe) + '/indices/names'
767
+ response = self.get(
768
+ r_url,
769
+ debug=debug
770
+ )
771
+ j = response.json()
772
+ if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
773
+ warn(j['detail'])
774
+ return None
775
+ if not isinstance(j, dict):
776
+ warn(response.text)
777
+ return None
778
+ return j
779
+
@@ -69,6 +69,10 @@ class SQLConnector(Connector):
69
69
  get_pipe_schema,
70
70
  create_pipe_table_from_df,
71
71
  get_pipe_columns_indices,
72
+ get_temporary_target,
73
+ create_pipe_indices,
74
+ drop_pipe_indices,
75
+ get_pipe_index_names,
72
76
  )
73
77
  from ._plugins import (
74
78
  register_plugin,
@@ -221,7 +225,7 @@ class SQLConnector(Connector):
221
225
  return None
222
226
 
223
227
  from meerschaum.utils.packages import attempt_import
224
- sqlalchemy_orm = attempt_import('sqlalchemy.orm')
228
+ sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
225
229
  session_factory = sqlalchemy_orm.sessionmaker(self.engine)
226
230
  self._Session = sqlalchemy_orm.scoped_session(session_factory)
227
231
 
@@ -289,7 +293,7 @@ class SQLConnector(Connector):
289
293
  Return the metadata bound to this configured schema.
290
294
  """
291
295
  from meerschaum.utils.packages import attempt_import
292
- sqlalchemy = attempt_import('sqlalchemy')
296
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
293
297
  if '_metadata' not in self.__dict__:
294
298
  self._metadata = sqlalchemy.MetaData(schema=self.schema)
295
299
  return self._metadata
@@ -367,7 +371,7 @@ class SQLConnector(Connector):
367
371
  self.__dict__['schema'] = None
368
372
  return None
369
373
 
370
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
374
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
371
375
  _schema = sqlalchemy.inspect(self.engine).default_schema_name
372
376
  self.__dict__['schema'] = _schema
373
377
  return _schema
@@ -186,7 +186,7 @@ def create_engine(
186
186
  """Create a sqlalchemy engine by building the engine string."""
187
187
  from meerschaum.utils.packages import attempt_import
188
188
  from meerschaum.utils.warnings import error, warn
189
- sqlalchemy = attempt_import('sqlalchemy')
189
+ sqlalchemy = attempt_import('sqlalchemy', lazy=False)
190
190
  import urllib
191
191
  import copy
192
192
  ### Install and patch required drivers.
@@ -148,7 +148,6 @@ def get_pipe_metadef(
148
148
  from meerschaum.utils.warnings import warn
149
149
  from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
150
150
  from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
151
- from meerschaum.utils.misc import is_int
152
151
  from meerschaum.config import get_config
153
152
 
154
153
  dt_col = pipe.columns.get('datetime', None)
@@ -191,7 +190,7 @@ def get_pipe_metadef(
191
190
  else begin
192
191
  )
193
192
 
194
- if begin and end and begin >= end:
193
+ if begin not in (None, '') and end is not None and begin >= end:
195
194
  begin = None
196
195
 
197
196
  if dt_name:
@@ -203,7 +202,7 @@ def get_pipe_metadef(
203
202
  begin=begin,
204
203
  db_type=db_dt_typ,
205
204
  )
206
- if begin
205
+ if begin not in ('', None)
207
206
  else None
208
207
  )
209
208
  end_da = (
@@ -214,7 +213,7 @@ def get_pipe_metadef(
214
213
  begin=end,
215
214
  db_type=db_dt_typ,
216
215
  )
217
- if end
216
+ if end is not None
218
217
  else None
219
218
  )
220
219
 
@@ -228,11 +227,7 @@ def get_pipe_metadef(
228
227
 
229
228
  has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
230
229
  if dt_name and (begin_da or end_da):
231
- definition_dt_name = (
232
- dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
233
- if not is_int((begin_da or end_da))
234
- else f"{definition_name}.{dt_name}"
235
- )
230
+ definition_dt_name = f"{definition_name}.{dt_name}"
236
231
  meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
237
232
  has_where = True
238
233
  if begin_da:
@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
25
25
  """
26
26
  from meerschaum.utils.misc import items_str
27
27
  from meerschaum.connectors.sql.tables import get_tables
28
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
28
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
29
29
  temp_tables_table = get_tables(
30
30
  mrsm_instance=self,
31
31
  create=create,
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
86
86
  """
87
87
  from meerschaum.utils.misc import items_str
88
88
  from meerschaum.connectors.sql.tables import get_tables
89
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
89
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
90
90
  temp_tables_table = get_tables(
91
91
  mrsm_instance=self,
92
92
  create=False,
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
150
150
  """
151
151
  from meerschaum.config import get_config
152
152
  from meerschaum.connectors.sql.tables import get_tables
153
- sqlalchemy = mrsm.attempt_import('sqlalchemy')
153
+ sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
154
154
  temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
155
155
  last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
156
156
  now_ts = time.perf_counter()