meerschaum 2.2.4__py3-none-any.whl → 2.2.5.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +23 -14
  2. meerschaum/_internal/arguments/_parser.py +4 -1
  3. meerschaum/_internal/entry.py +2 -4
  4. meerschaum/_internal/shell/Shell.py +0 -3
  5. meerschaum/actions/__init__.py +5 -1
  6. meerschaum/actions/backup.py +43 -0
  7. meerschaum/actions/bootstrap.py +32 -7
  8. meerschaum/actions/delete.py +62 -0
  9. meerschaum/actions/edit.py +98 -15
  10. meerschaum/actions/python.py +44 -2
  11. meerschaum/actions/show.py +26 -0
  12. meerschaum/actions/uninstall.py +24 -29
  13. meerschaum/api/_oauth2.py +17 -0
  14. meerschaum/api/routes/_login.py +23 -7
  15. meerschaum/config/__init__.py +16 -6
  16. meerschaum/config/_edit.py +1 -1
  17. meerschaum/config/_paths.py +3 -0
  18. meerschaum/config/_version.py +1 -1
  19. meerschaum/config/stack/__init__.py +3 -1
  20. meerschaum/core/Pipe/_fetch.py +25 -21
  21. meerschaum/core/Pipe/_sync.py +89 -59
  22. meerschaum/plugins/bootstrap.py +333 -0
  23. meerschaum/utils/daemon/Daemon.py +14 -3
  24. meerschaum/utils/daemon/FileDescriptorInterceptor.py +21 -14
  25. meerschaum/utils/daemon/RotatingFile.py +21 -18
  26. meerschaum/utils/formatting/__init__.py +22 -10
  27. meerschaum/utils/packages/_packages.py +1 -1
  28. meerschaum/utils/prompt.py +64 -21
  29. meerschaum/utils/yaml.py +32 -1
  30. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/METADATA +5 -2
  31. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/RECORD +37 -35
  32. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/WHEEL +1 -1
  33. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/LICENSE +0 -0
  34. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/NOTICE +0 -0
  35. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/entry_points.txt +0 -0
  36. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/top_level.txt +0 -0
  37. {meerschaum-2.2.4.dist-info → meerschaum-2.2.5.dev2.dist-info}/zip-safe +0 -0
@@ -25,9 +25,9 @@ def uninstall(
25
25
 
26
26
 
27
27
  def _complete_uninstall(
28
- action: Optional[List[str]] = None,
29
- **kw: Any
30
- ) -> List[str]:
28
+ action: Optional[List[str]] = None,
29
+ **kw: Any
30
+ ) -> List[str]:
31
31
  """
32
32
  Override the default Meerschaum `complete_` function.
33
33
  """
@@ -49,15 +49,16 @@ def _complete_uninstall(
49
49
  from meerschaum._internal.shell import default_action_completer
50
50
  return default_action_completer(action=(['uninstall'] + action), **kw)
51
51
 
52
+
52
53
  def _uninstall_plugins(
53
- action: Optional[List[str]] = None,
54
- repository: Optional[str] = None,
55
- yes: bool = False,
56
- force: bool = False,
57
- noask: bool = False,
58
- debug: bool = False,
59
- **kw: Any
60
- ) -> SuccessTuple:
54
+ action: Optional[List[str]] = None,
55
+ repository: Optional[str] = None,
56
+ yes: bool = False,
57
+ force: bool = False,
58
+ noask: bool = False,
59
+ debug: bool = False,
60
+ **kw: Any
61
+ ) -> SuccessTuple:
61
62
  """
62
63
  Remove installed plugins. Does not affect repository registrations.
63
64
  """
@@ -135,26 +136,20 @@ def _uninstall_plugins(
135
136
 
136
137
 
137
138
  def _complete_uninstall_plugins(action: Optional[List[str]] = None, **kw) -> List[str]:
138
- from meerschaum.plugins import get_plugins_names
139
- _plugin_names = get_plugins_names()
140
- if not action:
141
- return _plugin_names
142
- possibilities = []
143
- for name in _plugin_names:
144
- if name.startswith(action[0]) and action[0] != name:
145
- possibilities.append(name)
146
- return possibilities
139
+ from meerschaum.actions.edit import _complete_edit_plugins
140
+ return _complete_edit_plugins(action=action, **kw)
141
+
147
142
 
148
143
  def _uninstall_packages(
149
- action: Optional[List[str]] = None,
150
- sub_args: Optional[List[str]] = None,
151
- venv: Optional[str] = 'mrsm',
152
- yes: bool = False,
153
- force: bool = False,
154
- noask: bool = False,
155
- debug: bool = False,
156
- **kw: Any
157
- ) -> SuccessTuple:
144
+ action: Optional[List[str]] = None,
145
+ sub_args: Optional[List[str]] = None,
146
+ venv: Optional[str] = 'mrsm',
147
+ yes: bool = False,
148
+ force: bool = False,
149
+ noask: bool = False,
150
+ debug: bool = False,
151
+ **kw: Any
152
+ ) -> SuccessTuple:
158
153
  """
159
154
  Uninstall PyPI packages from the Meerschaum virtual environment.
160
155
 
meerschaum/api/_oauth2.py CHANGED
@@ -13,6 +13,23 @@ fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
13
13
  fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
14
14
  fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
15
15
 
16
+ class CustomOAuth2PasswordRequestForm:
17
+ def __init__(
18
+ self,
19
+ grant_type: str = fastapi.Form(None, regex="password|client_credentials"),
20
+ username: str = fastapi.Form(...),
21
+ password: str = fastapi.Form(...),
22
+ scope: str = fastapi.Form(""),
23
+ client_id: str = fastapi.Form(None),
24
+ client_secret: str = fastapi.Form(None),
25
+ ):
26
+ self.grant_type = grant_type
27
+ self.username = username
28
+ self.password = password
29
+ self.scope = scope
30
+ self.client_id = client_id
31
+ self.client_secret = client_secret
32
+
16
33
  LoginManager = fastapi_login.LoginManager
17
34
  def generate_secret_key() -> str:
18
35
  """
@@ -8,13 +8,17 @@ Manage access and refresh tokens.
8
8
 
9
9
  from datetime import datetime, timedelta, timezone
10
10
  import fastapi
11
+ from fastapi import Request, status
11
12
  from fastapi_login.exceptions import InvalidCredentialsException
12
- from fastapi.security import OAuth2PasswordRequestForm
13
+ from fastapi.exceptions import RequestValidationError
13
14
  from starlette.responses import Response, JSONResponse
14
15
  from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
15
16
  from meerschaum.core import User
16
17
  from meerschaum.config.static import STATIC_CONFIG
17
- from meerschaum.utils.typing import Dict, Any
18
+ from meerschaum.utils.typing import Dict, Any, Optional
19
+ from meerschaum.core.User._User import verify_password
20
+ from meerschaum.utils.warnings import warn
21
+ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
18
22
 
19
23
 
20
24
  @manager.user_loader()
@@ -28,18 +32,18 @@ def load_user(
28
32
 
29
33
 
30
34
  @app.post(endpoints['login'], tags=['Users'])
31
- def login(
32
- data: OAuth2PasswordRequestForm = fastapi.Depends()
35
+ async def login(
36
+ data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
37
+ # data: dict[str, str],
38
+ # request: Request
33
39
  ) -> Dict[str, Any]:
34
40
  """
35
41
  Login and set the session token.
36
42
  """
37
43
  username, password = (
38
- (data['username'], data['password']) if isinstance(data, dict)
39
- else (data.username, data.password)
44
+ (data.username, data.password)
40
45
  ) if not no_auth else ('no-auth', 'no-auth')
41
46
 
42
- from meerschaum.core.User._User import verify_password
43
47
  user = User(username, password)
44
48
  correct_password = no_auth or verify_password(
45
49
  password,
@@ -60,3 +64,15 @@ def login(
60
64
  'token_type': 'bearer',
61
65
  'expires' : expires_dt,
62
66
  }
67
+
68
+
69
+ @app.exception_handler(RequestValidationError)
70
+ async def validation_exception_handler(request: Request, exc: RequestValidationError):
71
+ """
72
+ Log validation errors as warnings.
73
+ """
74
+ warn(f"Validation error: {exc.errors()}", stack=False)
75
+ return JSONResponse(
76
+ status_code = status.HTTP_422_UNPROCESSABLE_ENTITY,
77
+ content = {"detail": exc.errors()},
78
+ )
@@ -12,7 +12,7 @@ from __future__ import annotations
12
12
  import os, shutil, sys, pathlib, copy
13
13
  from meerschaum.utils.typing import Any, Dict, Optional, Union
14
14
  from meerschaum.utils.threading import RLock
15
- from meerschaum.utils.warnings import warn
15
+ from meerschaum.utils.warnings import warn, error
16
16
 
17
17
  from meerschaum.config._version import __version__
18
18
  from meerschaum.config._edit import edit_config, write_config
@@ -239,17 +239,28 @@ def get_config(
239
239
  return c
240
240
 
241
241
 
242
- def get_plugin_config(*keys : str, **kw : Any) -> Optional[Any]:
242
+ def get_plugin_config(
243
+ *keys: str,
244
+ warn: bool = False,
245
+ **kw: Any
246
+ ) -> Optional[Any]:
243
247
  """
244
248
  This may only be called from within a Meerschaum plugin.
245
249
  See `meerschaum.config.get_config` for arguments.
246
250
  """
247
- from meerschaum.utils.warnings import warn, error
248
251
  from meerschaum.plugins import _get_parent_plugin
249
252
  parent_plugin_name = _get_parent_plugin(2)
250
253
  if parent_plugin_name is None:
251
- error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
252
- return get_config(*(['plugins', parent_plugin_name] + list(keys)), **kw)
254
+ error(
255
+ "You may only call `get_plugin_config()` "
256
+ "from within a Meerschaum plugin."
257
+ )
258
+
259
+ return get_config(
260
+ *(['plugins', parent_plugin_name] + list(keys)),
261
+ warn=warn,
262
+ **kw
263
+ )
253
264
 
254
265
 
255
266
  def write_plugin_config(
@@ -259,7 +270,6 @@ def write_plugin_config(
259
270
  """
260
271
  Write a plugin's configuration dictionary.
261
272
  """
262
- from meerschaum.utils.warnings import warn, error
263
273
  from meerschaum.plugins import _get_parent_plugin
264
274
  parent_plugin_name = _get_parent_plugin(2)
265
275
  if parent_plugin_name is None:
@@ -171,7 +171,7 @@ def general_write_yaml_config(
171
171
  path = pathlib.Path(fp)
172
172
  path.parent.mkdir(parents=True, exist_ok=True)
173
173
  path.touch(exist_ok=True)
174
- with open(path, 'w+') as f:
174
+ with open(path, 'w+', encoding='utf-8') as f:
175
175
  if header is not None:
176
176
  if debug:
177
177
  dprint(f"Header detected, writing to {path}...")
@@ -138,6 +138,9 @@ paths = {
138
138
  'SQLITE_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'sqlite'),
139
139
  'SQLITE_DB_PATH' : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
140
140
 
141
+ 'BACKUP_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'backup'),
142
+ 'BACKUP_DATABASE_PATH' : ('{BACKUP_RESOURCES_PATH}', 'backup.sql'),
143
+
141
144
  'DUCKDB_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', 'duckdb'),
142
145
  'DUCKDB_PATH' : ('{DUCKDB_RESOURCES_PATH}', 'duck.db'),
143
146
 
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.2.4"
5
+ __version__ = "2.2.5.dev2"
@@ -56,6 +56,7 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
56
56
 
57
57
  volumes = {
58
58
  'api_root': '/meerschaum',
59
+ 'api_user_local': '/home/meerschaum/.local',
59
60
  'meerschaum_db_data': '/var/lib/postgresql/data',
60
61
  'grafana_storage': '/var/lib/grafana',
61
62
  }
@@ -122,7 +123,7 @@ default_docker_compose_config = {
122
123
  ],
123
124
  'hostname' : f'{db_hostname}',
124
125
  'volumes' : [
125
- 'meerschaum_db_data' + ':' + volumes['meerschaum_db_data'],
126
+ 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
126
127
  ],
127
128
  'shm_size': '1024m',
128
129
  'networks' : [
@@ -159,6 +160,7 @@ default_docker_compose_config = {
159
160
  },
160
161
  'volumes' : [
161
162
  'api_root:' + volumes['api_root'],
163
+ 'api_user_local:' + volumes['api_user_local'],
162
164
  ],
163
165
  },
164
166
  'grafana': {
@@ -12,6 +12,7 @@ import meerschaum as mrsm
12
12
  from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
13
13
  from meerschaum.config import get_config
14
14
  from meerschaum.utils.warnings import warn
15
+ from meerschaum.utils.misc import filter_keywords
15
16
 
16
17
  def fetch(
17
18
  self,
@@ -75,25 +76,28 @@ def fetch(
75
76
  with mrsm.Venv(get_connector_plugin(self.connector)):
76
77
  df = self.connector.fetch(
77
78
  self,
78
- begin = _determine_begin(
79
- self,
80
- begin,
81
- check_existing = check_existing,
82
- debug = debug,
83
- ),
84
- end = end,
85
- chunk_hook = _chunk_hook,
86
- debug = debug,
87
- **kw
79
+ **filter_keywords(
80
+ self.connector.fetch,
81
+ begin=_determine_begin(
82
+ self,
83
+ begin,
84
+ check_existing=check_existing,
85
+ debug=debug,
86
+ ),
87
+ end=end,
88
+ chunk_hook=_chunk_hook,
89
+ debug=debug,
90
+ **kw
91
+ )
88
92
  )
89
93
  return df
90
94
 
91
95
 
92
96
  def get_backtrack_interval(
93
- self,
94
- check_existing: bool = True,
95
- debug: bool = False,
96
- ) -> Union[timedelta, int]:
97
+ self,
98
+ check_existing: bool = True,
99
+ debug: bool = False,
100
+ ) -> Union[timedelta, int]:
97
101
  """
98
102
  Get the chunk interval to use for this pipe.
99
103
 
@@ -127,17 +131,17 @@ def get_backtrack_interval(
127
131
 
128
132
 
129
133
  def _determine_begin(
130
- pipe: mrsm.Pipe,
131
- begin: Union[datetime, int, str] = '',
132
- check_existing: bool = True,
133
- debug: bool = False,
134
- ) -> Union[datetime, int, None]:
134
+ pipe: mrsm.Pipe,
135
+ begin: Union[datetime, int, str, None] = '',
136
+ check_existing: bool = True,
137
+ debug: bool = False,
138
+ ) -> Union[datetime, int, None]:
135
139
  """
136
140
  Apply the backtrack interval if `--begin` is not provided.
137
141
 
138
142
  Parameters
139
143
  ----------
140
- begin: Union[datetime, int, str], default ''
144
+ begin: Union[datetime, int, str, None], default ''
141
145
  The provided begin timestamp.
142
146
 
143
147
  check_existing: bool, default True
@@ -160,4 +164,4 @@ def _determine_begin(
160
164
  return sync_time - backtrack_interval
161
165
  except Exception as e:
162
166
  warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
163
- return sync_time
167
+ return sync_time
@@ -34,29 +34,29 @@ class InferFetch:
34
34
  MRSM_INFER_FETCH: bool = True
35
35
 
36
36
  def sync(
37
- self,
38
- df: Union[
39
- pd.DataFrame,
40
- Dict[str, List[Any]],
41
- List[Dict[str, Any]],
42
- InferFetch
43
- ] = InferFetch,
44
- begin: Union[datetime, int, str, None] = '',
45
- end: Union[datetime, int] = None,
46
- force: bool = False,
47
- retries: int = 10,
48
- min_seconds: int = 1,
49
- check_existing: bool = True,
50
- blocking: bool = True,
51
- workers: Optional[int] = None,
52
- callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
53
- error_callback: Optional[Callable[[Exception], Any]] = None,
54
- chunksize: Optional[int] = -1,
55
- sync_chunks: bool = True,
56
- debug: bool = False,
57
- _inplace: bool = True,
58
- **kw: Any
59
- ) -> SuccessTuple:
37
+ self,
38
+ df: Union[
39
+ pd.DataFrame,
40
+ Dict[str, List[Any]],
41
+ List[Dict[str, Any]],
42
+ InferFetch
43
+ ] = InferFetch,
44
+ begin: Union[datetime, int, str, None] = '',
45
+ end: Union[datetime, int, None] = None,
46
+ force: bool = False,
47
+ retries: int = 10,
48
+ min_seconds: int = 1,
49
+ check_existing: bool = True,
50
+ blocking: bool = True,
51
+ workers: Optional[int] = None,
52
+ callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
53
+ error_callback: Optional[Callable[[Exception], Any]] = None,
54
+ chunksize: Optional[int] = -1,
55
+ sync_chunks: bool = True,
56
+ debug: bool = False,
57
+ _inplace: bool = True,
58
+ **kw: Any
59
+ ) -> SuccessTuple:
60
60
  """
61
61
  Fetch new data from the source and update the pipe's table with new data.
62
62
 
@@ -125,7 +125,7 @@ def sync(
125
125
  from meerschaum.utils.formatting import get_console
126
126
  from meerschaum.utils.venv import Venv
127
127
  from meerschaum.connectors import get_connector_plugin
128
- from meerschaum.utils.misc import df_is_chunk_generator
128
+ from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
129
129
  from meerschaum.utils.pool import get_pool
130
130
  from meerschaum.config import get_config
131
131
 
@@ -186,7 +186,7 @@ def sync(
186
186
  ### use that instead.
187
187
  ### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
188
188
  ### If a DataFrame is provided, continue as expected.
189
- if hasattr(df, 'MRSM_INFER_FETCH'):
189
+ if hasattr(df, 'MRSM_INFER_FETCH'):
190
190
  try:
191
191
  if p.connector is None:
192
192
  msg = f"{p} does not have a valid connector."
@@ -194,7 +194,7 @@ def sync(
194
194
  msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
195
195
  p._exists = None
196
196
  return False, msg
197
- except Exception as e:
197
+ except Exception:
198
198
  p._exists = None
199
199
  return False, f"Unable to create the connector for {p}."
200
200
 
@@ -210,14 +210,28 @@ def sync(
210
210
  ):
211
211
  with Venv(get_connector_plugin(self.instance_connector)):
212
212
  p._exists = None
213
- return self.instance_connector.sync_pipe_inplace(p, debug=debug, **kw)
213
+ return self.instance_connector.sync_pipe_inplace(
214
+ p,
215
+ **filter_keywords(
216
+ p.instance_connector.sync_pipe_inplace,
217
+ debug=debug,
218
+ **kw
219
+ )
220
+ )
214
221
 
215
222
 
216
223
  ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
217
224
  try:
218
225
  if getattr(p.connector, 'sync', None) is not None:
219
226
  with Venv(get_connector_plugin(p.connector), debug=debug):
220
- return_tuple = p.connector.sync(p, debug=debug, **kw)
227
+ return_tuple = p.connector.sync(
228
+ p,
229
+ **filter_keywords(
230
+ p.connector.sync,
231
+ debug=debug,
232
+ **kw
233
+ )
234
+ )
221
235
  p._exists = None
222
236
  if not isinstance(return_tuple, tuple):
223
237
  return_tuple = (
@@ -237,13 +251,19 @@ def sync(
237
251
  ### Fetch the dataframe from the connector's `fetch()` method.
238
252
  try:
239
253
  with Venv(get_connector_plugin(p.connector), debug=debug):
240
- df = p.fetch(debug=debug, **kw)
254
+ df = p.fetch(
255
+ **filter_keywords(
256
+ p.fetch,
257
+ debug=debug,
258
+ **kw
259
+ )
260
+ )
241
261
 
242
262
  except Exception as e:
243
263
  get_console().print_exception(
244
- suppress = [
245
- 'meerschaum/core/Pipe/_sync.py',
246
- 'meerschaum/core/Pipe/_fetch.py',
264
+ suppress=[
265
+ 'meerschaum/core/Pipe/_sync.py',
266
+ 'meerschaum/core/Pipe/_fetch.py',
247
267
  ]
248
268
  )
249
269
  msg = f"Failed to fetch data from {p.connector}:\n {e}"
@@ -289,7 +309,7 @@ def sync(
289
309
  if not chunk_success:
290
310
  return chunk_success, f"Unable to sync initial chunk for {p}:\n{chunk_msg}"
291
311
  if debug:
292
- dprint(f"Successfully synced the first chunk, attemping the rest...")
312
+ dprint("Successfully synced the first chunk, attemping the rest...")
293
313
 
294
314
  failed_chunks = []
295
315
  def _process_chunk(_chunk):
@@ -309,7 +329,6 @@ def sync(
309
329
  )
310
330
  )
311
331
 
312
-
313
332
  results = sorted(
314
333
  [(chunk_success, chunk_msg)] + (
315
334
  list(pool.imap(_process_chunk, df))
@@ -329,7 +348,7 @@ def sync(
329
348
  retry_success = True
330
349
  if not success and any(success_bools):
331
350
  if debug:
332
- dprint(f"Retrying failed chunks...")
351
+ dprint("Retrying failed chunks...")
333
352
  chunks_to_retry = [c for c in failed_chunks]
334
353
  failed_chunks = []
335
354
  for chunk in chunks_to_retry:
@@ -361,9 +380,9 @@ def sync(
361
380
  while run:
362
381
  with Venv(get_connector_plugin(self.instance_connector)):
363
382
  return_tuple = p.instance_connector.sync_pipe(
364
- pipe = p,
365
- df = df,
366
- debug = debug,
383
+ pipe=p,
384
+ df=df,
385
+ debug=debug,
367
386
  **kw
368
387
  )
369
388
  _retries += 1
@@ -382,7 +401,7 @@ def sync(
382
401
  _checkpoint(**kw)
383
402
  if self.cache_pipe is not None:
384
403
  if debug:
385
- dprint(f"Caching retrieved dataframe.", **kw)
404
+ dprint("Caching retrieved dataframe.", **kw)
386
405
  _sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
387
406
  if not _sync_cache_tuple[0]:
388
407
  warn(f"Failed to sync local cache for {self}.")
@@ -395,10 +414,10 @@ def sync(
395
414
  return _sync(self, df = df)
396
415
 
397
416
  from meerschaum.utils.threading import Thread
398
- def default_callback(result_tuple : SuccessTuple):
417
+ def default_callback(result_tuple: SuccessTuple):
399
418
  dprint(f"Asynchronous result from {self}: {result_tuple}", **kw)
400
419
 
401
- def default_error_callback(x : Exception):
420
+ def default_error_callback(x: Exception):
402
421
  dprint(f"Error received for {self}: {x}", **kw)
403
422
 
404
423
  if callback is None and debug:
@@ -407,12 +426,12 @@ def sync(
407
426
  error_callback = default_error_callback
408
427
  try:
409
428
  thread = Thread(
410
- target = _sync,
411
- args = (self,),
412
- kwargs = {'df' : df},
413
- daemon = False,
414
- callback = callback,
415
- error_callback = error_callback
429
+ target=_sync,
430
+ args=(self,),
431
+ kwargs={'df': df},
432
+ daemon=False,
433
+ callback=callback,
434
+ error_callback=error_callback,
416
435
  )
417
436
  thread.start()
418
437
  except Exception as e:
@@ -424,12 +443,13 @@ def sync(
424
443
 
425
444
 
426
445
  def get_sync_time(
427
- self,
428
- params: Optional[Dict[str, Any]] = None,
429
- newest: bool = True,
430
- round_down: bool = False,
431
- debug: bool = False
432
- ) -> Union['datetime', None]:
446
+ self,
447
+ params: Optional[Dict[str, Any]] = None,
448
+ newest: bool = True,
449
+ apply_backtrack_interval: bool = False,
450
+ round_down: bool = False,
451
+ debug: bool = False
452
+ ) -> Union['datetime', None]:
433
453
  """
434
454
  Get the most recent datetime value for a Pipe.
435
455
 
@@ -443,6 +463,9 @@ def get_sync_time(
443
463
  If `True`, get the most recent datetime (honoring `params`).
444
464
  If `False`, get the oldest datetime (`ASC` instead of `DESC`).
445
465
 
466
+ apply_backtrack_interval: bool, default False
467
+ If `True`, subtract the backtrack interval from the sync time.
468
+
446
469
  round_down: bool, default False
447
470
  If `True`, round down the datetime value to the nearest minute.
448
471
 
@@ -461,15 +484,22 @@ def get_sync_time(
461
484
  with Venv(get_connector_plugin(self.instance_connector)):
462
485
  sync_time = self.instance_connector.get_sync_time(
463
486
  self,
464
- params = params,
465
- newest = newest,
466
- debug = debug,
487
+ params=params,
488
+ newest=newest,
489
+ debug=debug,
467
490
  )
468
491
 
469
- if not round_down or not isinstance(sync_time, datetime):
470
- return sync_time
492
+ if round_down and isinstance(sync_time, datetime):
493
+ sync_time = round_time(sync_time, timedelta(minutes=1))
494
+
495
+ if apply_backtrack_interval and sync_time is not None:
496
+ backtrack_interval = self.get_backtrack_interval(debug=debug)
497
+ try:
498
+ sync_time -= backtrack_interval
499
+ except Exception as e:
500
+ warn(f"Failed to apply backtrack interval:\n{e}")
471
501
 
472
- return round_time(sync_time, timedelta(minutes=1))
502
+ return sync_time
473
503
 
474
504
 
475
505
  def exists(