meerschaum 2.1.7__py3-none-any.whl → 2.2.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +3 -0
- meerschaum/_internal/entry.py +2 -1
- meerschaum/actions/install.py +7 -3
- meerschaum/actions/sync.py +7 -3
- meerschaum/api/dash/callbacks/dashboard.py +88 -13
- meerschaum/api/dash/callbacks/jobs.py +55 -3
- meerschaum/api/dash/jobs.py +34 -8
- meerschaum/api/dash/pipes.py +105 -18
- meerschaum/api/resources/static/js/xterm.js +1 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +0 -1
- meerschaum/connectors/api/_plugins.py +2 -1
- meerschaum/connectors/sql/_create_engine.py +5 -5
- meerschaum/plugins/_Plugin.py +11 -2
- meerschaum/utils/daemon/Daemon.py +11 -3
- meerschaum/utils/dtypes/__init__.py +9 -5
- meerschaum/utils/packages/__init__.py +4 -1
- meerschaum/utils/packages/_packages.py +6 -6
- meerschaum/utils/schedule.py +268 -29
- meerschaum/utils/typing.py +1 -1
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/METADATA +12 -14
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/RECORD +28 -28
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/LICENSE +0 -0
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/NOTICE +0 -0
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/WHEEL +0 -0
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/top_level.txt +0 -0
- {meerschaum-2.1.7.dist-info → meerschaum-2.2.0.dev2.dist-info}/zip-safe +0 -0
meerschaum/config/_version.py
CHANGED
@@ -49,6 +49,7 @@ def register_plugin(
|
|
49
49
|
def install_plugin(
|
50
50
|
self,
|
51
51
|
name: str,
|
52
|
+
skip_deps: bool = False,
|
52
53
|
force: bool = False,
|
53
54
|
debug: bool = False
|
54
55
|
) -> SuccessTuple:
|
@@ -78,7 +79,7 @@ def install_plugin(
|
|
78
79
|
success, msg = False, fail_msg
|
79
80
|
return success, msg
|
80
81
|
plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
|
81
|
-
return plugin.install(force=force, debug=debug)
|
82
|
+
return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
|
82
83
|
|
83
84
|
def get_plugins(
|
84
85
|
self,
|
@@ -154,10 +154,10 @@ install_flavor_drivers = {
|
|
154
154
|
'duckdb': ['duckdb', 'duckdb_engine'],
|
155
155
|
'mysql': ['pymysql'],
|
156
156
|
'mariadb': ['pymysql'],
|
157
|
-
'timescaledb': ['
|
158
|
-
'postgresql': ['
|
159
|
-
'citus': ['
|
160
|
-
'cockroachdb': ['
|
157
|
+
'timescaledb': ['psycopg'],
|
158
|
+
'postgresql': ['psycopg'],
|
159
|
+
'citus': ['psycopg'],
|
160
|
+
'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
|
161
161
|
'mssql': ['pyodbc'],
|
162
162
|
'oracle': ['cx_Oracle'],
|
163
163
|
}
|
@@ -165,7 +165,7 @@ require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalche
|
|
165
165
|
|
166
166
|
flavor_dialects = {
|
167
167
|
'cockroachdb': (
|
168
|
-
'cockroachdb', 'sqlalchemy_cockroachdb.
|
168
|
+
'cockroachdb', 'sqlalchemy_cockroachdb.psycopg', 'CockroachDBDialect_psycopg'
|
169
169
|
),
|
170
170
|
'duckdb': ('duckdb', 'duckdb_engine', 'Dialect'),
|
171
171
|
}
|
meerschaum/plugins/_Plugin.py
CHANGED
@@ -252,6 +252,7 @@ class Plugin:
|
|
252
252
|
|
253
253
|
def install(
|
254
254
|
self,
|
255
|
+
skip_deps: bool = False,
|
255
256
|
force: bool = False,
|
256
257
|
debug: bool = False,
|
257
258
|
) -> SuccessTuple:
|
@@ -263,6 +264,9 @@ class Plugin:
|
|
263
264
|
|
264
265
|
Parameters
|
265
266
|
----------
|
267
|
+
skip_deps: bool, default False
|
268
|
+
If `True`, do not install dependencies.
|
269
|
+
|
266
270
|
force: bool, default False
|
267
271
|
If `True`, continue with installation, even if required packages fail to install.
|
268
272
|
|
@@ -366,7 +370,11 @@ class Plugin:
|
|
366
370
|
plugin_installation_dir_path = path
|
367
371
|
break
|
368
372
|
|
369
|
-
success_msg =
|
373
|
+
success_msg = (
|
374
|
+
f"Successfully installed plugin '{self}'"
|
375
|
+
+ ("\n (skipped dependencies)" if skip_deps else "")
|
376
|
+
+ "."
|
377
|
+
)
|
370
378
|
success, abort = None, None
|
371
379
|
|
372
380
|
if is_same_version and not force:
|
@@ -423,7 +431,8 @@ class Plugin:
|
|
423
431
|
return success, msg
|
424
432
|
|
425
433
|
### attempt to install dependencies
|
426
|
-
|
434
|
+
dependencies_installed = skip_deps or self.install_dependencies(force=force, debug=debug)
|
435
|
+
if not dependencies_installed:
|
427
436
|
_ongoing_installations.remove(self.full_name)
|
428
437
|
return False, f"Failed to install dependencies for plugin '{self}'."
|
429
438
|
|
@@ -865,21 +865,29 @@ class Daemon:
|
|
865
865
|
error(_write_pickle_success_tuple[1])
|
866
866
|
|
867
867
|
|
868
|
-
def cleanup(self, keep_logs: bool = False) ->
|
869
|
-
"""
|
868
|
+
def cleanup(self, keep_logs: bool = False) -> SuccessTuple:
|
869
|
+
"""
|
870
|
+
Remove a daemon's directory after execution.
|
870
871
|
|
871
872
|
Parameters
|
872
873
|
----------
|
873
874
|
keep_logs: bool, default False
|
874
875
|
If `True`, skip deleting the daemon's log files.
|
876
|
+
|
877
|
+
Returns
|
878
|
+
-------
|
879
|
+
A `SuccessTuple` indicating success.
|
875
880
|
"""
|
876
881
|
if self.path.exists():
|
877
882
|
try:
|
878
883
|
shutil.rmtree(self.path)
|
879
884
|
except Exception as e:
|
880
|
-
|
885
|
+
msg = f"Failed to clean up '{self.daemon_id}':\n{e}"
|
886
|
+
warn(msg)
|
887
|
+
return False, msg
|
881
888
|
if not keep_logs:
|
882
889
|
self.rotating_log.delete()
|
890
|
+
return True, "Success"
|
883
891
|
|
884
892
|
|
885
893
|
def get_timeout_seconds(self, timeout: Union[int, float, None] = None) -> Union[int, float]:
|
@@ -6,8 +6,10 @@
|
|
6
6
|
Utility functions for working with data types.
|
7
7
|
"""
|
8
8
|
|
9
|
+
import traceback
|
9
10
|
from decimal import Decimal, Context, InvalidOperation
|
10
11
|
from meerschaum.utils.typing import Dict, Union, Any
|
12
|
+
from meerschaum.utils.warnings import warn
|
11
13
|
|
12
14
|
MRSM_PD_DTYPES: Dict[str, str] = {
|
13
15
|
'json': 'object',
|
@@ -37,9 +39,7 @@ def to_pandas_dtype(dtype: str) -> str:
|
|
37
39
|
from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
|
38
40
|
return get_pd_type_from_db_type(dtype)
|
39
41
|
|
40
|
-
import traceback
|
41
42
|
from meerschaum.utils.packages import attempt_import
|
42
|
-
from meerschaum.utils.warnings import warn
|
43
43
|
pandas = attempt_import('pandas', lazy=False)
|
44
44
|
|
45
45
|
try:
|
@@ -88,8 +88,12 @@ def are_dtypes_equal(
|
|
88
88
|
return False
|
89
89
|
return True
|
90
90
|
|
91
|
-
|
92
|
-
|
91
|
+
try:
|
92
|
+
if ldtype == rdtype:
|
93
|
+
return True
|
94
|
+
except Exception as e:
|
95
|
+
warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
|
96
|
+
return False
|
93
97
|
|
94
98
|
### Sometimes pandas dtype objects are passed.
|
95
99
|
ldtype = str(ldtype)
|
@@ -177,7 +181,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
|
|
177
181
|
return value
|
178
182
|
|
179
183
|
|
180
|
-
def value_is_null(value: Any) ->
|
184
|
+
def value_is_null(value: Any) -> bool:
|
181
185
|
"""
|
182
186
|
Determine if a value is a null-like string.
|
183
187
|
"""
|
@@ -8,7 +8,7 @@ Functions for managing packages and virtual environments reside here.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
|
-
import importlib.util, os, pathlib
|
11
|
+
import importlib.util, os, pathlib, re
|
12
12
|
from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
|
13
13
|
from meerschaum.utils.threading import Lock, RLock
|
14
14
|
from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
|
@@ -640,6 +640,9 @@ def need_update(
|
|
640
640
|
|
641
641
|
### We might be depending on a prerelease.
|
642
642
|
### Sanity check that the required version is not greater than the installed version.
|
643
|
+
if 'a' in required_version:
|
644
|
+
required_version = required_version.replace('a', '-dev')
|
645
|
+
version = version.replace('a', '-dev')
|
643
646
|
try:
|
644
647
|
return (
|
645
648
|
(not semver.Version.parse(version).match(required_version))
|
@@ -52,11 +52,11 @@ packages: Dict[str, Dict[str, str]] = {
|
|
52
52
|
'watchgod' : 'watchgod>=0.7.0',
|
53
53
|
'dill' : 'dill>=0.3.3',
|
54
54
|
'virtualenv' : 'virtualenv>=20.1.0',
|
55
|
-
'
|
55
|
+
'apscheduler' : 'apscheduler>=4.0.0a4',
|
56
56
|
},
|
57
57
|
'drivers': {
|
58
58
|
'cryptography' : 'cryptography>=38.0.1',
|
59
|
-
'
|
59
|
+
'psycopg' : 'psycopg[binary]>=3.1.18',
|
60
60
|
'pymysql' : 'PyMySQL>=0.9.0',
|
61
61
|
'aiomysql' : 'aiomysql>=0.0.21',
|
62
62
|
'sqlalchemy_cockroachdb' : 'sqlalchemy-cockroachdb>=2.0.0',
|
@@ -75,11 +75,11 @@ packages: Dict[str, Dict[str, str]] = {
|
|
75
75
|
'gadwall' : 'gadwall>=0.2.0',
|
76
76
|
},
|
77
77
|
'stack': {
|
78
|
-
'compose' : 'docker-compose>=1.
|
78
|
+
'compose' : 'docker-compose>=1.29.2',
|
79
79
|
},
|
80
80
|
'build': {
|
81
|
-
'cx_Freeze' : 'cx_Freeze>=
|
82
|
-
'PyInstaller' : 'pyinstaller
|
81
|
+
'cx_Freeze' : 'cx_Freeze>=7.0.0',
|
82
|
+
'PyInstaller' : 'pyinstaller>6.6.0',
|
83
83
|
},
|
84
84
|
'dev-tools': {
|
85
85
|
'twine' : 'twine>=3.2.0',
|
@@ -149,7 +149,7 @@ packages['api'] = {
|
|
149
149
|
'passlib' : 'passlib>=1.7.4',
|
150
150
|
'fastapi_login' : 'fastapi-login>=1.7.2',
|
151
151
|
'multipart' : 'python-multipart>=0.0.5',
|
152
|
-
'pydantic' : 'pydantic
|
152
|
+
# 'pydantic' : 'pydantic>2.0.0',
|
153
153
|
'httpx' : 'httpx>=0.24.1',
|
154
154
|
'websockets' : 'websockets>=11.0.3',
|
155
155
|
}
|
meerschaum/utils/schedule.py
CHANGED
@@ -7,11 +7,69 @@ Schedule processes and threads.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
10
|
+
import sys
|
11
|
+
from datetime import datetime, timezone, timedelta, timedelta
|
12
|
+
import meerschaum as mrsm
|
13
|
+
from meerschaum.utils.typing import Callable, Any, Optional, List, Dict
|
14
|
+
|
15
|
+
INTERVAL_UNITS: List[str] = ['months', 'weeks', 'days', 'hours', 'minutes', 'seconds']
|
16
|
+
FREQUENCY_ALIASES: Dict[str, str] = {
|
17
|
+
'daily': 'every 1 day',
|
18
|
+
'hourly': 'every 1 hour',
|
19
|
+
'minutely': 'every 1 minute',
|
20
|
+
'weekly': 'every 1 week',
|
21
|
+
'monthly': 'every 1 month',
|
22
|
+
'secondly': 'every 1 second',
|
23
|
+
}
|
24
|
+
LOGIC_ALIASES: Dict[str, str] = {
|
25
|
+
'and': '&',
|
26
|
+
'or': '|',
|
27
|
+
' through ': '-',
|
28
|
+
' thru ': '-',
|
29
|
+
' - ': '-',
|
30
|
+
'beginning': 'starting',
|
31
|
+
}
|
32
|
+
CRON_DAYS_OF_WEEK: List[str] = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
|
33
|
+
CRON_DAYS_OF_WEEK_ALIASES: Dict[str, str] = {
|
34
|
+
'monday': 'mon',
|
35
|
+
'tuesday': 'tue',
|
36
|
+
'tues': 'tue',
|
37
|
+
'wednesday': 'wed',
|
38
|
+
'thursday': 'thu',
|
39
|
+
'thurs': 'thu',
|
40
|
+
'friday': 'fri',
|
41
|
+
'saturday': 'sat',
|
42
|
+
'sunday': 'sun',
|
43
|
+
}
|
44
|
+
CRON_MONTHS: List[str] = [
|
45
|
+
'jan', 'feb', 'mar', 'apr', 'may', 'jun',
|
46
|
+
'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
|
47
|
+
]
|
48
|
+
CRON_MONTHS_ALIASES: Dict[str, str] = {
|
49
|
+
'january': 'jan',
|
50
|
+
'february': 'feb',
|
51
|
+
'march': 'mar',
|
52
|
+
'april': 'apr',
|
53
|
+
'may': 'may',
|
54
|
+
'june': 'jun',
|
55
|
+
'july': 'jul',
|
56
|
+
'august': 'aug',
|
57
|
+
'september': 'sep',
|
58
|
+
'october': 'oct',
|
59
|
+
'november': 'nov',
|
60
|
+
'december': 'dec',
|
61
|
+
}
|
62
|
+
SCHEDULE_ALIASES: Dict[str, str] = {
|
63
|
+
**FREQUENCY_ALIASES,
|
64
|
+
**LOGIC_ALIASES,
|
65
|
+
**CRON_DAYS_OF_WEEK_ALIASES,
|
66
|
+
**CRON_MONTHS_ALIASES,
|
67
|
+
}
|
68
|
+
STARTING_KEYWORD: str = 'starting'
|
11
69
|
|
12
70
|
def schedule_function(
|
13
71
|
function: Callable[[Any], Any],
|
14
|
-
|
72
|
+
schedule: str,
|
15
73
|
*args,
|
16
74
|
debug: bool = False,
|
17
75
|
**kw
|
@@ -25,41 +83,222 @@ def schedule_function(
|
|
25
83
|
function: Callable[[Any], Any]
|
26
84
|
The function to execute.
|
27
85
|
|
28
|
-
|
29
|
-
The frequency at which `function` should be executed (e.g. `'daily'`).
|
86
|
+
schedule: str
|
87
|
+
The frequency schedule at which `function` should be executed (e.g. `'daily'`).
|
30
88
|
|
31
89
|
"""
|
32
90
|
import warnings
|
33
91
|
from meerschaum.utils.warnings import warn
|
34
|
-
from meerschaum.utils.
|
35
|
-
from meerschaum.utils.misc import filter_keywords
|
36
|
-
from concurrent.futures._base import CancelledError
|
92
|
+
from meerschaum.utils.misc import filter_keywords, round_time
|
37
93
|
kw['debug'] = debug
|
38
94
|
kw = filter_keywords(function, **kw)
|
39
95
|
|
40
|
-
|
41
|
-
|
96
|
+
apscheduler = mrsm.attempt_import('apscheduler', lazy=False)
|
97
|
+
now = round_time(datetime.now(timezone.utc), timedelta(minutes=1))
|
98
|
+
trigger = parse_schedule(schedule, now=now)
|
42
99
|
|
43
|
-
|
44
|
-
|
45
|
-
try:
|
46
|
-
app = rocketry.Rocketry()
|
47
|
-
FuncTask = rocketry.tasks.FuncTask
|
48
|
-
with warnings.catch_warnings():
|
49
|
-
warnings.filterwarnings('ignore', 'Task\'s session not defined.')
|
50
|
-
task = FuncTask(_wrapper, start_cond=frequency)
|
51
|
-
app.session.add_task(task)
|
52
|
-
return app.run(debug=debug)
|
53
|
-
except (KeyboardInterrupt, CancelledError):
|
100
|
+
with apscheduler.Scheduler() as scheduler:
|
101
|
+
job = scheduler.add_schedule(function, trigger, args=args, kwargs=kw)
|
54
102
|
try:
|
55
|
-
|
56
|
-
except
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
103
|
+
scheduler.run_until_stopped()
|
104
|
+
except KeyboardInterrupt as e:
|
105
|
+
scheduler.stop()
|
106
|
+
scheduler.wait_until_stopped()
|
107
|
+
|
108
|
+
|
109
|
+
def parse_schedule(schedule: str, now: Optional[datetime] = None):
|
110
|
+
"""
|
111
|
+
Parse a schedule string (e.g. 'daily') into a Trigger object.
|
112
|
+
"""
|
113
|
+
from meerschaum.utils.warnings import error
|
114
|
+
from meerschaum.utils.misc import items_str, is_int
|
115
|
+
(
|
116
|
+
apscheduler_triggers_cron,
|
117
|
+
apscheduler_triggers_interval,
|
118
|
+
apscheduler_triggers_calendarinterval,
|
119
|
+
apscheduler_triggers_combining,
|
120
|
+
) = (
|
121
|
+
mrsm.attempt_import(
|
122
|
+
'apscheduler.triggers.cron',
|
123
|
+
'apscheduler.triggers.interval',
|
124
|
+
'apscheduler.triggers.calendarinterval',
|
125
|
+
'apscheduler.triggers.combining',
|
126
|
+
lazy = False,
|
64
127
|
)
|
128
|
+
)
|
129
|
+
|
130
|
+
starting_ts = parse_start_time(schedule, now=now)
|
131
|
+
schedule = schedule.split(STARTING_KEYWORD)[0].strip()
|
132
|
+
for alias_keyword, true_keyword in SCHEDULE_ALIASES.items():
|
133
|
+
schedule = schedule.replace(alias_keyword, true_keyword)
|
134
|
+
|
135
|
+
### TODO Allow for combining `and` + `or` logic.
|
136
|
+
if '&' in schedule and '|' in schedule:
|
137
|
+
error(f"Cannot accept both 'and' + 'or' logic in the schedule frequency.", ValueError)
|
138
|
+
|
139
|
+
join_str = '|' if '|' in schedule else '&'
|
140
|
+
join_trigger = (
|
141
|
+
apscheduler_triggers_combining.OrTrigger
|
142
|
+
if join_str == '|'
|
143
|
+
else apscheduler_triggers_combining.AndTrigger
|
144
|
+
)
|
145
|
+
join_kwargs = {
|
146
|
+
'max_iterations': 1_000_000,
|
147
|
+
'threshold': 0,
|
148
|
+
} if join_str == '&' else {}
|
149
|
+
|
150
|
+
schedule_parts = [part.strip() for part in schedule.split(join_str)]
|
151
|
+
triggers = []
|
152
|
+
|
153
|
+
has_seconds = 'second' in schedule
|
154
|
+
has_minutes = 'minute' in schedule
|
155
|
+
has_days = 'day' in schedule
|
156
|
+
has_weeks = 'week' in schedule
|
157
|
+
has_hours = 'hour' in schedule
|
158
|
+
num_hourly_intervals = schedule.count('hour')
|
159
|
+
divided_days = False
|
160
|
+
divided_hours = False
|
161
|
+
|
162
|
+
for schedule_part in schedule_parts:
|
163
|
+
|
164
|
+
### Intervals must begin with 'every' (after alias substitution).
|
165
|
+
if schedule_part.lower().startswith('every '):
|
166
|
+
schedule_num_str, schedule_unit = (
|
167
|
+
schedule_part[len('every '):].split(' ', maxsplit=1)
|
168
|
+
)
|
169
|
+
schedule_unit = schedule_unit.rstrip('s') + 's'
|
170
|
+
if schedule_unit not in INTERVAL_UNITS:
|
171
|
+
error(
|
172
|
+
f"Invalid interval '{schedule_unit}'.\n"
|
173
|
+
+ f" Accepted values are {items_str(INTERVAL_UNITS)}.",
|
174
|
+
ValueError,
|
175
|
+
)
|
176
|
+
|
177
|
+
schedule_num = (
|
178
|
+
int(schedule_num_str)
|
179
|
+
if is_int(schedule_num_str)
|
180
|
+
else float(schedule_num_str)
|
181
|
+
)
|
182
|
+
|
183
|
+
### NOTE: When combining days or weeks with other schedules,
|
184
|
+
### we must divide one of the day-schedules by 2.
|
185
|
+
### TODO Remove this when APScheduler is patched.
|
186
|
+
if (
|
187
|
+
join_str == '&'
|
188
|
+
and (has_days or has_weeks)
|
189
|
+
and len(schedule_parts) > 1
|
190
|
+
and not divided_days
|
191
|
+
):
|
192
|
+
schedule_num /= 2
|
193
|
+
divided_days = True
|
65
194
|
|
195
|
+
### NOTE: When combining multiple hourly intervals,
|
196
|
+
### one must be divided by 2.
|
197
|
+
if (
|
198
|
+
join_str == '&'
|
199
|
+
# and num_hourly_intervals > 1
|
200
|
+
and len(schedule_parts) > 1
|
201
|
+
and not divided_hours
|
202
|
+
):
|
203
|
+
print("divided hours")
|
204
|
+
schedule_num /= 2
|
205
|
+
# divided_hours = True
|
206
|
+
|
207
|
+
trigger = (
|
208
|
+
apscheduler_triggers_interval.IntervalTrigger(
|
209
|
+
**{
|
210
|
+
schedule_unit: schedule_num,
|
211
|
+
'start_time': starting_ts,
|
212
|
+
}
|
213
|
+
)
|
214
|
+
if schedule_unit != 'months' else (
|
215
|
+
apscheduler_triggers_calendarinterval.CalendarIntervalTrigger(
|
216
|
+
**{
|
217
|
+
schedule_unit: schedule_num,
|
218
|
+
'start_date': starting_ts,
|
219
|
+
# 'timezone': starting_ts.tzinfo, TODO Re-enable once APScheduler updates.
|
220
|
+
}
|
221
|
+
)
|
222
|
+
)
|
223
|
+
)
|
224
|
+
|
225
|
+
### Determine whether this is a pure cron string or a cron subset (e.g. 'may-aug')_.
|
226
|
+
else:
|
227
|
+
first_three_prefix = schedule_part[:3]
|
228
|
+
cron_kw = {}
|
229
|
+
if first_three_prefix in CRON_DAYS_OF_WEEK:
|
230
|
+
cron_kw['day_of_week'] = schedule_part
|
231
|
+
elif first_three_prefix in CRON_MONTHS:
|
232
|
+
cron_kw['month'] = schedule_part
|
233
|
+
trigger = (
|
234
|
+
apscheduler_triggers_cron.CronTrigger(
|
235
|
+
**{
|
236
|
+
**cron_kw,
|
237
|
+
'hour': '*',
|
238
|
+
'minute': '*' if has_minutes else starting_ts.minute,
|
239
|
+
'second': '*' if has_seconds else starting_ts.second,
|
240
|
+
'start_time': starting_ts,
|
241
|
+
'timezone': starting_ts.tzinfo,
|
242
|
+
}
|
243
|
+
)
|
244
|
+
if cron_kw
|
245
|
+
else apscheduler_triggers_cron.CronTrigger.from_crontab(
|
246
|
+
schedule_part,
|
247
|
+
timezone = starting_ts.tzinfo,
|
248
|
+
)
|
249
|
+
)
|
250
|
+
### Explicitly set the `start_time` after building with `from_crontab`.
|
251
|
+
if trigger.start_time != starting_ts:
|
252
|
+
trigger.start_time = starting_ts
|
253
|
+
|
254
|
+
triggers.append(trigger)
|
255
|
+
|
256
|
+
return (
|
257
|
+
join_trigger(triggers, **join_kwargs)
|
258
|
+
if len(triggers) != 1
|
259
|
+
else triggers[0]
|
260
|
+
)
|
261
|
+
|
262
|
+
|
263
|
+
def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
|
264
|
+
"""
|
265
|
+
Return the datetime to use for the given schedule string.
|
266
|
+
|
267
|
+
Parameters
|
268
|
+
----------
|
269
|
+
schedule: str
|
270
|
+
The schedule frequency to be parsed into a starting datetime.
|
271
|
+
|
272
|
+
now: Optional[datetime], default None
|
273
|
+
If provided, use this value as a default if no start time is explicitly stated.
|
274
|
+
|
275
|
+
Returns
|
276
|
+
-------
|
277
|
+
A `datetime` object, either `now` or the datetime embedded in the schedule string.
|
278
|
+
|
279
|
+
Examples
|
280
|
+
--------
|
281
|
+
>>> parse_start_time('daily starting 2024-01-01')
|
282
|
+
datetime.datetime(2024, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)
|
283
|
+
>>> parse_start_time('monthly starting 1st')
|
284
|
+
datetime.datetime(2024, 5, 1, 0, 0, tzinfo=datetime.timezone.utc)
|
285
|
+
>>> parse_start_time('hourly starting 00:30')
|
286
|
+
datetime.datetime(2024, 5, 13, 0, 30, tzinfo=datetime.timezone.utc)
|
287
|
+
"""
|
288
|
+
from meerschaum.utils.misc import round_time
|
289
|
+
from meerschaum.utils.warnings import error, warn
|
290
|
+
dateutil_parser = mrsm.attempt_import('dateutil.parser')
|
291
|
+
starting_parts = schedule.split(STARTING_KEYWORD)
|
292
|
+
starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()
|
293
|
+
now = now or round_time(datetime.now(timezone.utc), timedelta(minutes=1))
|
294
|
+
try:
|
295
|
+
starting_ts = now if starting_str == 'now' else dateutil_parser.parse(starting_str)
|
296
|
+
schedule_parse_error = None
|
297
|
+
except Exception as e:
|
298
|
+
warn(f"Unable to parse starting time from '{starting_str}'.", stack=False)
|
299
|
+
schedule_parse_error = str(e)
|
300
|
+
if schedule_parse_error:
|
301
|
+
error(schedule_parse_error, ValueError, stack=False)
|
302
|
+
if not starting_ts.tzinfo:
|
303
|
+
starting_ts = starting_ts.replace(tzinfo=timezone.utc)
|
304
|
+
return starting_ts
|
meerschaum/utils/typing.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: meerschaum
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.2.0.dev2
|
4
4
|
Summary: Sync Time-Series Pipes with Meerschaum
|
5
5
|
Home-page: https://meerschaum.io
|
6
6
|
Author: Bennett Meares
|
@@ -56,7 +56,7 @@ Requires-Dist: psutil >=5.8.0 ; extra == '_required'
|
|
56
56
|
Requires-Dist: watchgod >=0.7.0 ; extra == '_required'
|
57
57
|
Requires-Dist: dill >=0.3.3 ; extra == '_required'
|
58
58
|
Requires-Dist: virtualenv >=20.1.0 ; extra == '_required'
|
59
|
-
Requires-Dist:
|
59
|
+
Requires-Dist: apscheduler >=4.0.0a4 ; extra == '_required'
|
60
60
|
Provides-Extra: api
|
61
61
|
Requires-Dist: uvicorn[standard] >=0.22.0 ; extra == 'api'
|
62
62
|
Requires-Dist: gunicorn >=20.1.0 ; extra == 'api'
|
@@ -66,7 +66,6 @@ Requires-Dist: fastapi >=0.100.0 ; extra == 'api'
|
|
66
66
|
Requires-Dist: passlib >=1.7.4 ; extra == 'api'
|
67
67
|
Requires-Dist: fastapi-login >=1.7.2 ; extra == 'api'
|
68
68
|
Requires-Dist: python-multipart >=0.0.5 ; extra == 'api'
|
69
|
-
Requires-Dist: pydantic <2.0.0 ; extra == 'api'
|
70
69
|
Requires-Dist: httpx >=0.24.1 ; extra == 'api'
|
71
70
|
Requires-Dist: numpy >=1.18.5 ; extra == 'api'
|
72
71
|
Requires-Dist: pandas[parquet] >=2.0.1 ; extra == 'api'
|
@@ -79,7 +78,7 @@ Requires-Dist: databases >=0.4.0 ; extra == 'api'
|
|
79
78
|
Requires-Dist: aiosqlite >=0.16.0 ; extra == 'api'
|
80
79
|
Requires-Dist: asyncpg >=0.21.0 ; extra == 'api'
|
81
80
|
Requires-Dist: cryptography >=38.0.1 ; extra == 'api'
|
82
|
-
Requires-Dist:
|
81
|
+
Requires-Dist: psycopg[binary] >=3.1.18 ; extra == 'api'
|
83
82
|
Requires-Dist: PyMySQL >=0.9.0 ; extra == 'api'
|
84
83
|
Requires-Dist: aiomysql >=0.0.21 ; extra == 'api'
|
85
84
|
Requires-Dist: sqlalchemy-cockroachdb >=2.0.0 ; extra == 'api'
|
@@ -106,7 +105,7 @@ Requires-Dist: psutil >=5.8.0 ; extra == 'api'
|
|
106
105
|
Requires-Dist: watchgod >=0.7.0 ; extra == 'api'
|
107
106
|
Requires-Dist: dill >=0.3.3 ; extra == 'api'
|
108
107
|
Requires-Dist: virtualenv >=20.1.0 ; extra == 'api'
|
109
|
-
Requires-Dist:
|
108
|
+
Requires-Dist: apscheduler >=4.0.0a4 ; extra == 'api'
|
110
109
|
Requires-Dist: pprintpp >=0.4.0 ; extra == 'api'
|
111
110
|
Requires-Dist: asciitree >=0.3.3 ; extra == 'api'
|
112
111
|
Requires-Dist: typing-extensions >=4.7.1 ; extra == 'api'
|
@@ -124,8 +123,8 @@ Requires-Dist: dash-daq >=0.5.0 ; extra == 'api'
|
|
124
123
|
Requires-Dist: terminado >=0.12.1 ; extra == 'api'
|
125
124
|
Requires-Dist: tornado >=6.1.0 ; extra == 'api'
|
126
125
|
Provides-Extra: build
|
127
|
-
Requires-Dist: cx-Freeze >=
|
128
|
-
Requires-Dist: pyinstaller
|
126
|
+
Requires-Dist: cx-Freeze >=7.0.0 ; extra == 'build'
|
127
|
+
Requires-Dist: pyinstaller >6.6.0 ; extra == 'build'
|
129
128
|
Provides-Extra: cli
|
130
129
|
Requires-Dist: pgcli >=3.1.0 ; extra == 'cli'
|
131
130
|
Requires-Dist: mycli >=1.23.2 ; extra == 'cli'
|
@@ -161,7 +160,7 @@ Requires-Dist: mkdocs-redirects >=1.0.4 ; extra == 'docs'
|
|
161
160
|
Requires-Dist: jinja2 ==3.0.3 ; extra == 'docs'
|
162
161
|
Provides-Extra: drivers
|
163
162
|
Requires-Dist: cryptography >=38.0.1 ; extra == 'drivers'
|
164
|
-
Requires-Dist:
|
163
|
+
Requires-Dist: psycopg[binary] >=3.1.18 ; extra == 'drivers'
|
165
164
|
Requires-Dist: PyMySQL >=0.9.0 ; extra == 'drivers'
|
166
165
|
Requires-Dist: aiomysql >=0.0.21 ; extra == 'drivers'
|
167
166
|
Requires-Dist: sqlalchemy-cockroachdb >=2.0.0 ; extra == 'drivers'
|
@@ -212,9 +211,9 @@ Requires-Dist: psutil >=5.8.0 ; extra == 'full'
|
|
212
211
|
Requires-Dist: watchgod >=0.7.0 ; extra == 'full'
|
213
212
|
Requires-Dist: dill >=0.3.3 ; extra == 'full'
|
214
213
|
Requires-Dist: virtualenv >=20.1.0 ; extra == 'full'
|
215
|
-
Requires-Dist:
|
214
|
+
Requires-Dist: apscheduler >=4.0.0a4 ; extra == 'full'
|
216
215
|
Requires-Dist: cryptography >=38.0.1 ; extra == 'full'
|
217
|
-
Requires-Dist:
|
216
|
+
Requires-Dist: psycopg[binary] >=3.1.18 ; extra == 'full'
|
218
217
|
Requires-Dist: PyMySQL >=0.9.0 ; extra == 'full'
|
219
218
|
Requires-Dist: aiomysql >=0.0.21 ; extra == 'full'
|
220
219
|
Requires-Dist: sqlalchemy-cockroachdb >=2.0.0 ; extra == 'full'
|
@@ -249,7 +248,6 @@ Requires-Dist: fastapi >=0.100.0 ; extra == 'full'
|
|
249
248
|
Requires-Dist: passlib >=1.7.4 ; extra == 'full'
|
250
249
|
Requires-Dist: fastapi-login >=1.7.2 ; extra == 'full'
|
251
250
|
Requires-Dist: python-multipart >=0.0.5 ; extra == 'full'
|
252
|
-
Requires-Dist: pydantic <2.0.0 ; extra == 'full'
|
253
251
|
Requires-Dist: httpx >=0.24.1 ; extra == 'full'
|
254
252
|
Provides-Extra: gui
|
255
253
|
Requires-Dist: toga >=0.3.0-dev29 ; extra == 'gui'
|
@@ -270,7 +268,7 @@ Requires-Dist: databases >=0.4.0 ; extra == 'sql'
|
|
270
268
|
Requires-Dist: aiosqlite >=0.16.0 ; extra == 'sql'
|
271
269
|
Requires-Dist: asyncpg >=0.21.0 ; extra == 'sql'
|
272
270
|
Requires-Dist: cryptography >=38.0.1 ; extra == 'sql'
|
273
|
-
Requires-Dist:
|
271
|
+
Requires-Dist: psycopg[binary] >=3.1.18 ; extra == 'sql'
|
274
272
|
Requires-Dist: PyMySQL >=0.9.0 ; extra == 'sql'
|
275
273
|
Requires-Dist: aiomysql >=0.0.21 ; extra == 'sql'
|
276
274
|
Requires-Dist: sqlalchemy-cockroachdb >=2.0.0 ; extra == 'sql'
|
@@ -297,9 +295,9 @@ Requires-Dist: psutil >=5.8.0 ; extra == 'sql'
|
|
297
295
|
Requires-Dist: watchgod >=0.7.0 ; extra == 'sql'
|
298
296
|
Requires-Dist: dill >=0.3.3 ; extra == 'sql'
|
299
297
|
Requires-Dist: virtualenv >=20.1.0 ; extra == 'sql'
|
300
|
-
Requires-Dist:
|
298
|
+
Requires-Dist: apscheduler >=4.0.0a4 ; extra == 'sql'
|
301
299
|
Provides-Extra: stack
|
302
|
-
Requires-Dist: docker-compose >=1.
|
300
|
+
Requires-Dist: docker-compose >=1.29.2 ; extra == 'stack'
|
303
301
|
|
304
302
|
<img src="https://meerschaum.io/assets/banner_1920x320.png" alt="Meerschaum banner" style="width: 100%"/>
|
305
303
|
|