meerschaum 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. meerschaum/__init__.py +4 -1
  2. meerschaum/_internal/arguments/_parse_arguments.py +23 -14
  3. meerschaum/_internal/arguments/_parser.py +4 -2
  4. meerschaum/_internal/docs/index.py +513 -110
  5. meerschaum/_internal/entry.py +2 -4
  6. meerschaum/_internal/shell/Shell.py +0 -3
  7. meerschaum/actions/__init__.py +5 -1
  8. meerschaum/actions/bootstrap.py +32 -7
  9. meerschaum/actions/delete.py +62 -0
  10. meerschaum/actions/edit.py +98 -15
  11. meerschaum/actions/python.py +45 -14
  12. meerschaum/actions/show.py +39 -4
  13. meerschaum/actions/stack.py +12 -12
  14. meerschaum/actions/uninstall.py +24 -29
  15. meerschaum/api/__init__.py +0 -1
  16. meerschaum/api/_oauth2.py +17 -0
  17. meerschaum/api/dash/__init__.py +0 -1
  18. meerschaum/api/dash/callbacks/custom.py +1 -1
  19. meerschaum/api/dash/plugins.py +5 -6
  20. meerschaum/api/routes/_login.py +23 -7
  21. meerschaum/config/__init__.py +16 -6
  22. meerschaum/config/_edit.py +1 -1
  23. meerschaum/config/_paths.py +3 -0
  24. meerschaum/config/_version.py +1 -1
  25. meerschaum/config/stack/__init__.py +3 -1
  26. meerschaum/connectors/Connector.py +7 -2
  27. meerschaum/connectors/__init__.py +7 -5
  28. meerschaum/core/Pipe/_data.py +23 -15
  29. meerschaum/core/Pipe/_deduplicate.py +1 -1
  30. meerschaum/core/Pipe/_dtypes.py +5 -0
  31. meerschaum/core/Pipe/_fetch.py +26 -20
  32. meerschaum/core/Pipe/_sync.py +96 -61
  33. meerschaum/plugins/__init__.py +1 -1
  34. meerschaum/plugins/bootstrap.py +333 -0
  35. meerschaum/utils/daemon/Daemon.py +14 -3
  36. meerschaum/utils/daemon/FileDescriptorInterceptor.py +21 -14
  37. meerschaum/utils/daemon/RotatingFile.py +21 -18
  38. meerschaum/utils/dataframe.py +12 -4
  39. meerschaum/utils/debug.py +9 -15
  40. meerschaum/utils/formatting/__init__.py +23 -10
  41. meerschaum/utils/misc.py +117 -11
  42. meerschaum/utils/packages/_packages.py +1 -0
  43. meerschaum/utils/prompt.py +64 -21
  44. meerschaum/utils/typing.py +1 -0
  45. meerschaum/utils/warnings.py +9 -1
  46. meerschaum/utils/yaml.py +32 -1
  47. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/METADATA +5 -1
  48. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/RECORD +54 -53
  49. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/WHEEL +1 -1
  50. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/LICENSE +0 -0
  51. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/NOTICE +0 -0
  52. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/entry_points.txt +0 -0
  53. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/top_level.txt +0 -0
  54. {meerschaum-2.2.3.dist-info → meerschaum-2.2.5.dist-info}/zip-safe +0 -0
meerschaum/api/_oauth2.py CHANGED
@@ -13,6 +13,23 @@ fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
13
13
  fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
14
14
  fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
15
15
 
16
+ class CustomOAuth2PasswordRequestForm:
17
+ def __init__(
18
+ self,
19
+ grant_type: str = fastapi.Form(None, regex="password|client_credentials"),
20
+ username: str = fastapi.Form(...),
21
+ password: str = fastapi.Form(...),
22
+ scope: str = fastapi.Form(""),
23
+ client_id: str = fastapi.Form(None),
24
+ client_secret: str = fastapi.Form(None),
25
+ ):
26
+ self.grant_type = grant_type
27
+ self.username = username
28
+ self.password = password
29
+ self.scope = scope
30
+ self.client_id = client_id
31
+ self.client_secret = client_secret
32
+
16
33
  LoginManager = fastapi_login.LoginManager
17
34
  def generate_secret_key() -> str:
18
35
  """
@@ -50,7 +50,6 @@ stylesheets = [
50
50
  '/static/css/bootstrap.min.css',
51
51
  '/static/css/dbc_dark.css',
52
52
  '/static/css/dash.css',
53
- # '/static/js/node_modules/xterm/css/xterm.css',
54
53
  ]
55
54
  scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
56
55
  dash_app = enrich.DashProxy(
@@ -21,7 +21,7 @@ def init_dash_plugins():
21
21
  for _function in _functions:
22
22
  try:
23
23
  _function(dash_app)
24
- except Exception as e:
24
+ except Exception:
25
25
  warn(
26
26
  f"Failed to load function '{_function.__name__}' "
27
27
  + f"from plugin '{_module_name}':\n"
@@ -7,7 +7,7 @@ Functions for interacting with plugins via the web interface.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import List, Tuple, SuccessTuple
10
+ from meerschaum.utils.typing import List, Tuple, SuccessTuple, Optional, WebState, Dict, Any
11
11
  from meerschaum.utils.packages import import_dcc, import_html
12
12
  from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
13
13
  html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
@@ -17,10 +17,10 @@ from meerschaum.api.dash import dash_app, debug, active_sessions
17
17
 
18
18
 
19
19
  def get_plugins_cards(
20
- state: Optional[WebState] = None,
21
- search_term: Optional[str] = None,
22
- session_data: Optional[Dict[str, Any]] = None,
23
- ) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
20
+ state: Optional[WebState] = None,
21
+ search_term: Optional[str] = None,
22
+ session_data: Optional[Dict[str, Any]] = None,
23
+ ) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
24
24
  """
25
25
  Return the cards and alerts for plugins.
26
26
  """
@@ -96,4 +96,3 @@ def is_plugin_owner(plugin_name: str, session_data: Dict['str', Any]) -> bool:
96
96
  _username is not None
97
97
  and _username == _plugin_username
98
98
  )
99
-
@@ -8,13 +8,17 @@ Manage access and refresh tokens.
8
8
 
9
9
  from datetime import datetime, timedelta, timezone
10
10
  import fastapi
11
+ from fastapi import Request, status
11
12
  from fastapi_login.exceptions import InvalidCredentialsException
12
- from fastapi.security import OAuth2PasswordRequestForm
13
+ from fastapi.exceptions import RequestValidationError
13
14
  from starlette.responses import Response, JSONResponse
14
15
  from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
15
16
  from meerschaum.core import User
16
17
  from meerschaum.config.static import STATIC_CONFIG
17
- from meerschaum.utils.typing import Dict, Any
18
+ from meerschaum.utils.typing import Dict, Any, Optional
19
+ from meerschaum.core.User._User import verify_password
20
+ from meerschaum.utils.warnings import warn
21
+ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
18
22
 
19
23
 
20
24
  @manager.user_loader()
@@ -28,18 +32,18 @@ def load_user(
28
32
 
29
33
 
30
34
  @app.post(endpoints['login'], tags=['Users'])
31
- def login(
32
- data: OAuth2PasswordRequestForm = fastapi.Depends()
35
+ async def login(
36
+ data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
37
+ # data: dict[str, str],
38
+ # request: Request
33
39
  ) -> Dict[str, Any]:
34
40
  """
35
41
  Login and set the session token.
36
42
  """
37
43
  username, password = (
38
- (data['username'], data['password']) if isinstance(data, dict)
39
- else (data.username, data.password)
44
+ (data.username, data.password)
40
45
  ) if not no_auth else ('no-auth', 'no-auth')
41
46
 
42
- from meerschaum.core.User._User import verify_password
43
47
  user = User(username, password)
44
48
  correct_password = no_auth or verify_password(
45
49
  password,
@@ -60,3 +64,15 @@ def login(
60
64
  'token_type': 'bearer',
61
65
  'expires' : expires_dt,
62
66
  }
67
+
68
+
69
+ @app.exception_handler(RequestValidationError)
70
+ async def validation_exception_handler(request: Request, exc: RequestValidationError):
71
+ """
72
+ Log validation errors as warnings.
73
+ """
74
+ warn(f"Validation error: {exc.errors()}", stack=False)
75
+ return JSONResponse(
76
+ status_code = status.HTTP_422_UNPROCESSABLE_ENTITY,
77
+ content = {"detail": exc.errors()},
78
+ )
@@ -12,7 +12,7 @@ from __future__ import annotations
12
12
  import os, shutil, sys, pathlib, copy
13
13
  from meerschaum.utils.typing import Any, Dict, Optional, Union
14
14
  from meerschaum.utils.threading import RLock
15
- from meerschaum.utils.warnings import warn
15
+ from meerschaum.utils.warnings import warn, error
16
16
 
17
17
  from meerschaum.config._version import __version__
18
18
  from meerschaum.config._edit import edit_config, write_config
@@ -239,17 +239,28 @@ def get_config(
239
239
  return c
240
240
 
241
241
 
242
- def get_plugin_config(*keys : str, **kw : Any) -> Optional[Any]:
242
+ def get_plugin_config(
243
+ *keys: str,
244
+ warn: bool = False,
245
+ **kw: Any
246
+ ) -> Optional[Any]:
243
247
  """
244
248
  This may only be called from within a Meerschaum plugin.
245
249
  See `meerschaum.config.get_config` for arguments.
246
250
  """
247
- from meerschaum.utils.warnings import warn, error
248
251
  from meerschaum.plugins import _get_parent_plugin
249
252
  parent_plugin_name = _get_parent_plugin(2)
250
253
  if parent_plugin_name is None:
251
- error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
252
- return get_config(*(['plugins', parent_plugin_name] + list(keys)), **kw)
254
+ error(
255
+ "You may only call `get_plugin_config()` "
256
+ "from within a Meerschaum plugin."
257
+ )
258
+
259
+ return get_config(
260
+ *(['plugins', parent_plugin_name] + list(keys)),
261
+ warn=warn,
262
+ **kw
263
+ )
253
264
 
254
265
 
255
266
  def write_plugin_config(
@@ -259,7 +270,6 @@ def write_plugin_config(
259
270
  """
260
271
  Write a plugin's configuration dictionary.
261
272
  """
262
- from meerschaum.utils.warnings import warn, error
263
273
  from meerschaum.plugins import _get_parent_plugin
264
274
  parent_plugin_name = _get_parent_plugin(2)
265
275
  if parent_plugin_name is None:
@@ -171,7 +171,7 @@ def general_write_yaml_config(
171
171
  path = pathlib.Path(fp)
172
172
  path.parent.mkdir(parents=True, exist_ok=True)
173
173
  path.touch(exist_ok=True)
174
- with open(path, 'w+') as f:
174
+ with open(path, 'w+', encoding='utf-8') as f:
175
175
  if header is not None:
176
176
  if debug:
177
177
  dprint(f"Header detected, writing to {path}...")
@@ -138,6 +138,9 @@ paths = {
138
138
  'SQLITE_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'sqlite'),
139
139
  'SQLITE_DB_PATH' : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
140
140
 
141
+ 'BACKUP_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'backup'),
142
+ 'BACKUP_DATABASE_PATH' : ('{BACKUP_RESOURCES_PATH}', 'backup.sql'),
143
+
141
144
  'DUCKDB_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'duckdb'),
142
145
  'DUCKDB_PATH' : ('{DUCKDB_RESOURCES_PATH}', 'duck.db'),
143
146
 
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.2.3"
5
+ __version__ = "2.2.5"
@@ -56,6 +56,7 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
56
56
 
57
57
  volumes = {
58
58
  'api_root': '/meerschaum',
59
+ 'api_user_local': '/home/meerschaum/.local',
59
60
  'meerschaum_db_data': '/var/lib/postgresql/data',
60
61
  'grafana_storage': '/var/lib/grafana',
61
62
  }
@@ -122,7 +123,7 @@ default_docker_compose_config = {
122
123
  ],
123
124
  'hostname' : f'{db_hostname}',
124
125
  'volumes' : [
125
- 'meerschaum_db_data' + ':' + volumes['meerschaum_db_data'],
126
+ 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
126
127
  ],
127
128
  'shm_size': '1024m',
128
129
  'networks' : [
@@ -159,6 +160,7 @@ default_docker_compose_config = {
159
160
  },
160
161
  'volumes' : [
161
162
  'api_root:' + volumes['api_root'],
163
+ 'api_user_local:' + volumes['api_user_local'],
162
164
  ],
163
165
  },
164
166
  'grafana': {
@@ -18,7 +18,7 @@ class InvalidAttributesError(Exception):
18
18
 
19
19
  class Connector(metaclass=abc.ABCMeta):
20
20
  """
21
- The base connector class to hold connection attributes,
21
+ The base connector class to hold connection attributes.
22
22
  """
23
23
  def __init__(
24
24
  self,
@@ -27,6 +27,8 @@ class Connector(metaclass=abc.ABCMeta):
27
27
  **kw: Any
28
28
  ):
29
29
  """
30
+ Set the given keyword arguments as attributes.
31
+
30
32
  Parameters
31
33
  ----------
32
34
  type: str
@@ -35,9 +37,12 @@ class Connector(metaclass=abc.ABCMeta):
35
37
  label: str
36
38
  The `label` for the connector.
37
39
 
40
+
41
+ Examples
42
+ --------
38
43
  Run `mrsm edit config` and to edit connectors in the YAML file:
39
44
 
40
- ```
45
+ ```yaml
41
46
  meerschaum:
42
47
  connections:
43
48
  {type}:
@@ -24,6 +24,7 @@ from meerschaum.connectors.api.APIConnector import APIConnector
24
24
  from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
25
25
 
26
26
  __all__ = (
27
+ "make_connector",
27
28
  "Connector",
28
29
  "SQLConnector",
29
30
  "APIConnector",
@@ -290,13 +291,14 @@ def make_connector(
290
291
  --------
291
292
  >>> import meerschaum as mrsm
292
293
  >>> from meerschaum.connectors import make_connector, Connector
294
+ >>>
295
+ >>> @make_connector
293
296
  >>> class FooConnector(Connector):
294
- ... def __init__(self, label: str, **kw):
295
- ... super().__init__('foo', label, **kw)
297
+ ... REQUIRED_ATTRIBUTES: list[str] = ['username', 'password']
296
298
  ...
297
- >>> make_connector(FooConnector)
298
- >>> mrsm.get_connector('foo', 'bar')
299
- foo:bar
299
+ >>> conn = mrsm.get_connector('foo:bar', username='dog', password='cat')
300
+ >>> print(conn.username, conn.password)
301
+ dog cat
300
302
  >>>
301
303
  """
302
304
  import re
@@ -8,24 +8,32 @@ Retrieve Pipes' data from instances.
8
8
 
9
9
  from __future__ import annotations
10
10
  from datetime import datetime, timedelta
11
- from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
11
+
12
+ import meerschaum as mrsm
13
+ from meerschaum.utils.typing import (
14
+ Optional, Dict, Any, Union, List, Tuple, Iterator, TYPE_CHECKING,
15
+ )
12
16
  from meerschaum.config import get_config
13
17
 
18
+ if TYPE_CHECKING:
19
+ pd = mrsm.attempt_import('pandas')
20
+
21
+
14
22
  def get_data(
15
- self,
16
- select_columns: Optional[List[str]] = None,
17
- omit_columns: Optional[List[str]] = None,
18
- begin: Union[datetime, int, None] = None,
19
- end: Union[datetime, int, None] = None,
20
- params: Optional[Dict[str, Any]] = None,
21
- as_iterator: bool = False,
22
- as_chunks: bool = False,
23
- as_dask: bool = False,
24
- chunk_interval: Union[timedelta, int, None] = None,
25
- fresh: bool = False,
26
- debug: bool = False,
27
- **kw: Any
28
- ) -> Union['pd.DataFrame', Generator['pd.DataFrame'], None]:
23
+ self,
24
+ select_columns: Optional[List[str]] = None,
25
+ omit_columns: Optional[List[str]] = None,
26
+ begin: Union[datetime, int, None] = None,
27
+ end: Union[datetime, int, None] = None,
28
+ params: Optional[Dict[str, Any]] = None,
29
+ as_iterator: bool = False,
30
+ as_chunks: bool = False,
31
+ as_dask: bool = False,
32
+ chunk_interval: Union[timedelta, int, None] = None,
33
+ fresh: bool = False,
34
+ debug: bool = False,
35
+ **kw: Any
36
+ ) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
29
37
  """
30
38
  Get a pipe's data from the instance connector.
31
39
 
@@ -8,7 +8,7 @@ Delete duplicate rows within a pipe's table.
8
8
 
9
9
  from __future__ import annotations
10
10
  from datetime import datetime, timedelta
11
- from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple
11
+ from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple, Union
12
12
 
13
13
 
14
14
  def deduplicate(
@@ -8,7 +8,12 @@ Enforce data types for a pipe's underlying table.
8
8
 
9
9
  from __future__ import annotations
10
10
  from io import StringIO
11
+ import meerschaum as mrsm
11
12
  from meerschaum.utils.typing import Dict, Any, Optional
13
+ from typing import TYPE_CHECKING
14
+
15
+ if TYPE_CHECKING:
16
+ pd = mrsm.attempt_import('pandas')
12
17
 
13
18
  def enforce_dtypes(
14
19
  self,
@@ -8,11 +8,15 @@ Functions for fetching new data into the Pipe
8
8
 
9
9
  from __future__ import annotations
10
10
  from datetime import timedelta, datetime
11
+
11
12
  import meerschaum as mrsm
12
- from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
13
+ from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
13
14
  from meerschaum.config import get_config
14
15
  from meerschaum.utils.warnings import warn
15
16
 
17
+ if TYPE_CHECKING:
18
+ pd = mrsm.attempt_import('pandas')
19
+
16
20
  def fetch(
17
21
  self,
18
22
  begin: Union[datetime, str, None] = '',
@@ -54,6 +58,7 @@ def fetch(
54
58
 
55
59
  from meerschaum.connectors import custom_types, get_connector_plugin
56
60
  from meerschaum.utils.debug import dprint, _checkpoint
61
+ from meerschaum.utils.misc import filter_arguments
57
62
 
58
63
  _chunk_hook = kw.pop('chunk_hook', None)
59
64
  kw['workers'] = self.get_num_workers(kw.get('workers', None))
@@ -71,29 +76,30 @@ def fetch(
71
76
  chunk_message = '\n' + chunk_label + '\n' + chunk_message
72
77
  return chunk_success, chunk_message
73
78
 
74
-
75
79
  with mrsm.Venv(get_connector_plugin(self.connector)):
76
- df = self.connector.fetch(
80
+ _args, _kwargs = filter_arguments(
81
+ self.connector.fetch,
77
82
  self,
78
- begin = _determine_begin(
83
+ begin=_determine_begin(
79
84
  self,
80
85
  begin,
81
- check_existing = check_existing,
82
- debug = debug,
86
+ check_existing=check_existing,
87
+ debug=debug,
83
88
  ),
84
- end = end,
85
- chunk_hook = _chunk_hook,
86
- debug = debug,
89
+ end=end,
90
+ chunk_hook=_chunk_hook,
91
+ debug=debug,
87
92
  **kw
88
93
  )
94
+ df = self.connector.fetch(*_args, **_kwargs)
89
95
  return df
90
96
 
91
97
 
92
98
  def get_backtrack_interval(
93
- self,
94
- check_existing: bool = True,
95
- debug: bool = False,
96
- ) -> Union[timedelta, int]:
99
+ self,
100
+ check_existing: bool = True,
101
+ debug: bool = False,
102
+ ) -> Union[timedelta, int]:
97
103
  """
98
104
  Get the chunk interval to use for this pipe.
99
105
 
@@ -127,17 +133,17 @@ def get_backtrack_interval(
127
133
 
128
134
 
129
135
  def _determine_begin(
130
- pipe: mrsm.Pipe,
131
- begin: Union[datetime, int, str] = '',
132
- check_existing: bool = True,
133
- debug: bool = False,
134
- ) -> Union[datetime, int, None]:
136
+ pipe: mrsm.Pipe,
137
+ begin: Union[datetime, int, str, None] = '',
138
+ check_existing: bool = True,
139
+ debug: bool = False,
140
+ ) -> Union[datetime, int, None]:
135
141
  """
136
142
  Apply the backtrack interval if `--begin` is not provided.
137
143
 
138
144
  Parameters
139
145
  ----------
140
- begin: Union[datetime, int, str], default ''
146
+ begin: Union[datetime, int, str, None], default ''
141
147
  The provided begin timestamp.
142
148
 
143
149
  check_existing: bool, default True
@@ -160,4 +166,4 @@ def _determine_begin(
160
166
  return sync_time - backtrack_interval
161
167
  except Exception as e:
162
168
  warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
163
- return sync_time
169
+ return sync_time