meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +198 -118
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +382 -95
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/js/terminado.js +3 -0
  68. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  69. meerschaum/api/resources/templates/termpage.html +13 -0
  70. meerschaum/api/routes/__init__.py +1 -0
  71. meerschaum/api/routes/_actions.py +3 -4
  72. meerschaum/api/routes/_connectors.py +3 -7
  73. meerschaum/api/routes/_jobs.py +26 -35
  74. meerschaum/api/routes/_login.py +120 -15
  75. meerschaum/api/routes/_misc.py +5 -10
  76. meerschaum/api/routes/_pipes.py +178 -143
  77. meerschaum/api/routes/_plugins.py +38 -28
  78. meerschaum/api/routes/_tokens.py +236 -0
  79. meerschaum/api/routes/_users.py +47 -35
  80. meerschaum/api/routes/_version.py +3 -3
  81. meerschaum/api/routes/_webterm.py +3 -3
  82. meerschaum/config/__init__.py +100 -30
  83. meerschaum/config/_default.py +132 -64
  84. meerschaum/config/_edit.py +38 -32
  85. meerschaum/config/_formatting.py +2 -0
  86. meerschaum/config/_patch.py +10 -8
  87. meerschaum/config/_paths.py +133 -13
  88. meerschaum/config/_read_config.py +87 -36
  89. meerschaum/config/_sync.py +6 -3
  90. meerschaum/config/_version.py +1 -1
  91. meerschaum/config/environment.py +262 -0
  92. meerschaum/config/stack/__init__.py +37 -15
  93. meerschaum/config/static.py +18 -0
  94. meerschaum/connectors/_Connector.py +11 -6
  95. meerschaum/connectors/__init__.py +41 -22
  96. meerschaum/connectors/api/_APIConnector.py +34 -6
  97. meerschaum/connectors/api/_actions.py +2 -2
  98. meerschaum/connectors/api/_jobs.py +12 -1
  99. meerschaum/connectors/api/_login.py +33 -7
  100. meerschaum/connectors/api/_misc.py +2 -2
  101. meerschaum/connectors/api/_pipes.py +23 -32
  102. meerschaum/connectors/api/_plugins.py +2 -2
  103. meerschaum/connectors/api/_request.py +1 -1
  104. meerschaum/connectors/api/_tokens.py +146 -0
  105. meerschaum/connectors/api/_users.py +70 -58
  106. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  107. meerschaum/connectors/instance/__init__.py +10 -0
  108. meerschaum/connectors/instance/_pipes.py +442 -0
  109. meerschaum/connectors/instance/_plugins.py +159 -0
  110. meerschaum/connectors/instance/_tokens.py +317 -0
  111. meerschaum/connectors/instance/_users.py +188 -0
  112. meerschaum/connectors/parse.py +5 -2
  113. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  114. meerschaum/connectors/sql/_cli.py +12 -11
  115. meerschaum/connectors/sql/_create_engine.py +12 -168
  116. meerschaum/connectors/sql/_fetch.py +2 -18
  117. meerschaum/connectors/sql/_pipes.py +295 -278
  118. meerschaum/connectors/sql/_plugins.py +29 -0
  119. meerschaum/connectors/sql/_sql.py +46 -21
  120. meerschaum/connectors/sql/_users.py +36 -2
  121. meerschaum/connectors/sql/tables/__init__.py +254 -122
  122. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  123. meerschaum/connectors/valkey/_pipes.py +60 -31
  124. meerschaum/connectors/valkey/_plugins.py +2 -26
  125. meerschaum/core/Pipe/__init__.py +115 -85
  126. meerschaum/core/Pipe/_attributes.py +425 -124
  127. meerschaum/core/Pipe/_bootstrap.py +54 -24
  128. meerschaum/core/Pipe/_cache.py +555 -0
  129. meerschaum/core/Pipe/_clear.py +0 -11
  130. meerschaum/core/Pipe/_data.py +96 -68
  131. meerschaum/core/Pipe/_deduplicate.py +0 -13
  132. meerschaum/core/Pipe/_delete.py +12 -21
  133. meerschaum/core/Pipe/_drop.py +11 -23
  134. meerschaum/core/Pipe/_dtypes.py +49 -19
  135. meerschaum/core/Pipe/_edit.py +14 -4
  136. meerschaum/core/Pipe/_fetch.py +1 -1
  137. meerschaum/core/Pipe/_index.py +8 -14
  138. meerschaum/core/Pipe/_show.py +5 -5
  139. meerschaum/core/Pipe/_sync.py +123 -204
  140. meerschaum/core/Pipe/_verify.py +4 -4
  141. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  142. meerschaum/core/Plugin/__init__.py +1 -1
  143. meerschaum/core/Token/_Token.py +220 -0
  144. meerschaum/core/Token/__init__.py +12 -0
  145. meerschaum/core/User/_User.py +35 -10
  146. meerschaum/core/User/__init__.py +9 -1
  147. meerschaum/core/__init__.py +1 -0
  148. meerschaum/jobs/_Executor.py +88 -4
  149. meerschaum/jobs/_Job.py +149 -38
  150. meerschaum/jobs/__init__.py +3 -2
  151. meerschaum/jobs/systemd.py +8 -3
  152. meerschaum/models/__init__.py +35 -0
  153. meerschaum/models/pipes.py +247 -0
  154. meerschaum/models/tokens.py +38 -0
  155. meerschaum/models/users.py +26 -0
  156. meerschaum/plugins/__init__.py +301 -88
  157. meerschaum/plugins/bootstrap.py +510 -4
  158. meerschaum/utils/_get_pipes.py +97 -30
  159. meerschaum/utils/daemon/Daemon.py +199 -43
  160. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  161. meerschaum/utils/daemon/RotatingFile.py +63 -36
  162. meerschaum/utils/daemon/StdinFile.py +53 -13
  163. meerschaum/utils/daemon/__init__.py +47 -6
  164. meerschaum/utils/daemon/_names.py +6 -3
  165. meerschaum/utils/dataframe.py +479 -81
  166. meerschaum/utils/debug.py +49 -19
  167. meerschaum/utils/dtypes/__init__.py +476 -34
  168. meerschaum/utils/dtypes/sql.py +369 -29
  169. meerschaum/utils/formatting/__init__.py +5 -2
  170. meerschaum/utils/formatting/_jobs.py +1 -1
  171. meerschaum/utils/formatting/_pipes.py +52 -50
  172. meerschaum/utils/formatting/_pprint.py +1 -0
  173. meerschaum/utils/formatting/_shell.py +44 -18
  174. meerschaum/utils/misc.py +268 -186
  175. meerschaum/utils/packages/__init__.py +25 -40
  176. meerschaum/utils/packages/_packages.py +42 -34
  177. meerschaum/utils/pipes.py +213 -0
  178. meerschaum/utils/process.py +2 -2
  179. meerschaum/utils/prompt.py +175 -144
  180. meerschaum/utils/schedule.py +2 -1
  181. meerschaum/utils/sql.py +134 -47
  182. meerschaum/utils/threading.py +42 -0
  183. meerschaum/utils/typing.py +1 -4
  184. meerschaum/utils/venv/_Venv.py +2 -2
  185. meerschaum/utils/venv/__init__.py +7 -7
  186. meerschaum/utils/warnings.py +19 -13
  187. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  188. meerschaum-3.0.0.dist-info/RECORD +289 -0
  189. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  190. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  191. meerschaum/api/models/_interfaces.py +0 -15
  192. meerschaum/api/models/_locations.py +0 -15
  193. meerschaum/api/models/_metrics.py +0 -15
  194. meerschaum/config/_environment.py +0 -145
  195. meerschaum/config/static/__init__.py +0 -186
  196. meerschaum-2.9.5.dist-info/RECORD +0 -263
  197. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  198. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  199. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  200. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -6,29 +6,79 @@
6
6
  Manage access and refresh tokens.
7
7
  """
8
8
 
9
+ import uuid
9
10
  from datetime import datetime, timedelta, timezone
11
+ from typing import Union
10
12
 
11
13
  import fastapi
12
- from fastapi import Request, status
14
+ from fastapi import Request, status, Response
13
15
  from fastapi_login.exceptions import InvalidCredentialsException
14
16
  from fastapi.exceptions import RequestValidationError
15
17
  from starlette.responses import JSONResponse
16
18
 
17
- from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
19
+ import meerschaum as mrsm
20
+ from meerschaum.api import (
21
+ endpoints,
22
+ get_api_connector,
23
+ get_cache_connector,
24
+ app,
25
+ debug,
26
+ manager,
27
+ no_auth,
28
+ )
18
29
  from meerschaum.core import User
19
- from meerschaum.config.static import STATIC_CONFIG
30
+ from meerschaum._internal.static import STATIC_CONFIG
20
31
  from meerschaum.utils.typing import Dict, Any
21
- from meerschaum.core.User._User import verify_password
32
+ from meerschaum.utils.misc import is_uuid, is_int
33
+ from meerschaum.core.User import verify_password
22
34
  from meerschaum.utils.warnings import warn
23
35
  from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
24
36
 
25
37
 
38
+ USER_ID_CACHE_EXPIRES_SECONDS: int = mrsm.get_config('system', 'api', 'cache', 'session_expires_minutes') * 60
39
+ _active_user_ids = {}
40
+
41
+
26
42
  @manager.user_loader()
27
- def load_user(username: str) -> User:
43
+ def load_user_or_token_from_manager(username_or_token_id: str) -> User:
28
44
  """
29
45
  Create the `meerschaum.core.User` object from the username.
30
46
  """
31
- return User(username, instance=get_api_connector())
47
+ cache_conn = get_cache_connector()
48
+ api_conn = get_api_connector()
49
+
50
+ is_token = is_uuid(username_or_token_id)
51
+
52
+ if is_token:
53
+ return api_conn.get_token(username_or_token_id)
54
+
55
+ username = username_or_token_id
56
+
57
+ cached_user_id = (
58
+ _active_user_ids.get(username)
59
+ if cache_conn is None
60
+ else cache_conn.get(f'mrsm:users:{username}:id')
61
+ )
62
+ if isinstance(cached_user_id, str):
63
+ if is_int(cached_user_id):
64
+ cached_user_id = int(cached_user_id)
65
+ elif is_uuid(cached_user_id):
66
+ cached_user_id = uuid.UUID(cached_user_id)
67
+
68
+ user = User(username, instance=api_conn, user_id=cached_user_id)
69
+
70
+ if cached_user_id is not None:
71
+ return user
72
+
73
+ user_id = api_conn.get_user_id(user)
74
+ if user_id is not None:
75
+ user._user_id = user_id
76
+ if cache_conn is not None:
77
+ cache_conn.set(f'mrsm:users:{username}:id', str(user_id), ex=USER_ID_CACHE_EXPIRES_SECONDS)
78
+ else:
79
+ _active_user_ids[username] = user_id
80
+ return user
81
+
32
82
 
33
83
 
34
84
  @app.post(endpoints['login'], tags=['Users'])
@@ -39,26 +89,81 @@ def login(
39
89
  Login and set the session token.
40
90
  """
41
91
  username, password = (
42
- (data['username'], data['password'])
92
+ (data.get('username', None), data.get('password', None))
43
93
  if isinstance(data, dict)
44
94
  else (data.username, data.password)
45
- ) if not no_auth else ('no-auth', 'no-auth')
46
-
47
- user = User(username, password)
48
- correct_password = no_auth or verify_password(
49
- password,
50
- get_api_connector().get_user_password_hash(user, debug=debug)
51
95
  )
52
- if not correct_password:
96
+ client_id, client_secret = (
97
+ (data.get('client_id', None), data.get('client_secret', None))
98
+ if isinstance(data, dict)
99
+ else (data.client_id, data.client_secret)
100
+ )
101
+ grant_type = (
102
+ data.get('grant_type', None)
103
+ if isinstance(data, dict)
104
+ else data.grant_type
105
+ )
106
+ if not grant_type:
107
+ grant_type = (
108
+ 'password'
109
+ if username and password
110
+ else 'client_credentials'
111
+ )
112
+
113
+ allowed_scopes = []
114
+ type_ = None
115
+ expires_dt: Union[datetime, None] = None
116
+ sub_id = None
117
+ if grant_type == 'password':
118
+ user = User(str(username), str(password), instance=get_api_connector())
119
+ correct_password = no_auth or verify_password(
120
+ str(password),
121
+ get_api_connector().get_user_password_hash(user, debug=debug)
122
+ )
123
+ if not correct_password:
124
+ raise InvalidCredentialsException
125
+
126
+ allowed_scopes = user.get_scopes(debug=debug)
127
+ type_ = get_api_connector().get_user_type(user, debug=debug)
128
+ sub_id = username
129
+
130
+ elif grant_type == 'client_credentials':
131
+ if not is_uuid(str(client_id)):
132
+ raise InvalidCredentialsException
133
+ token_id = uuid.UUID(client_id)
134
+ correct_password = no_auth or verify_password(
135
+ str(client_secret),
136
+ str(get_api_connector().get_token_secret_hash(token_id, debug=debug))
137
+ )
138
+ if not correct_password:
139
+ raise InvalidCredentialsException
140
+
141
+ allowed_scopes = get_api_connector().get_token_scopes(token_id, debug=debug)
142
+ sub_id = client_id
143
+
144
+ else:
53
145
  raise InvalidCredentialsException
54
146
 
147
+ requested_scopes = data.scope.split()
148
+ if '*' in allowed_scopes:
149
+ final_scopes = requested_scopes or ['*']
150
+ else:
151
+ final_scopes = [
152
+ s for s in requested_scopes if s in allowed_scopes
153
+ ] if requested_scopes else allowed_scopes
154
+
55
155
  expires_minutes = STATIC_CONFIG['api']['oauth']['token_expires_minutes']
56
156
  expires_delta = timedelta(minutes=expires_minutes)
57
157
  expires_dt = datetime.now(timezone.utc).replace(tzinfo=None) + expires_delta
58
158
  access_token = manager.create_access_token(
59
- data={'sub': username},
159
+ data={
160
+ 'sub': sub_id,
161
+ 'scopes': final_scopes,
162
+ 'type': type_,
163
+ },
60
164
  expires=expires_delta
61
165
  )
166
+
62
167
  return {
63
168
  'access_token': access_token,
64
169
  'token_type': 'bearer',
@@ -19,7 +19,8 @@ from meerschaum.api import (
19
19
  debug,
20
20
  get_api_connector,
21
21
  private,
22
- manager
22
+ manager,
23
+ ScopedAuth,
23
24
  )
24
25
  from meerschaum.config.paths import API_STATIC_PATH
25
26
  from meerschaum import __version__ as version
@@ -38,9 +39,7 @@ def get_favicon() -> Any:
38
39
 
39
40
  @app.get(endpoints['chaining'], tags=['Misc'])
40
41
  def get_chaining_status(
41
- curr_user = (
42
- fastapi.Depends(manager) if private else None
43
- ),
42
+ curr_user = fastapi.Depends(ScopedAuth(['instance:read'])) if private else None,
44
43
  ) -> bool:
45
44
  """
46
45
  Return whether this API instance may be chained.
@@ -50,9 +49,7 @@ def get_chaining_status(
50
49
 
51
50
  @app.get(endpoints['info'], tags=['Misc'])
52
51
  def get_instance_info(
53
- curr_user = (
54
- fastapi.Depends(manager) if private else None
55
- ),
52
+ curr_user = fastapi.Depends(ScopedAuth(['instance:read'])) if private else None,
56
53
  instance_keys: Optional[str] = None,
57
54
  ) -> Dict[str, Union[str, int]]:
58
55
  """
@@ -87,9 +84,7 @@ def get_healtheck(instance_keys: Optional[str] = None) -> Dict[str, Any]:
87
84
  if debug:
88
85
  @app.get('/id', tags=['Misc'])
89
86
  def get_ids(
90
- curr_user = (
91
- fastapi.Depends(manager) if private else None
92
- ),
87
+ curr_user = fastapi.Depends(ScopedAuth(['instance:read'])) if private else None,
93
88
  ) -> Dict[str, Union[int, str]]:
94
89
  return {
95
90
  'server': SERVER_ID,