FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/cfg/__init__.py +3 -3
- flowerpower/cfg/pipeline/__init__.py +5 -3
- flowerpower/cfg/project/__init__.py +3 -3
- flowerpower/cfg/project/job_queue.py +1 -128
- flowerpower/cli/__init__.py +5 -5
- flowerpower/cli/cfg.py +0 -3
- flowerpower/cli/job_queue.py +401 -133
- flowerpower/cli/pipeline.py +14 -413
- flowerpower/cli/utils.py +0 -1
- flowerpower/flowerpower.py +537 -28
- flowerpower/job_queue/__init__.py +5 -94
- flowerpower/job_queue/base.py +201 -3
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
- flowerpower/job_queue/rq/manager.py +388 -77
- flowerpower/pipeline/__init__.py +2 -0
- flowerpower/pipeline/base.py +2 -2
- flowerpower/pipeline/io.py +14 -16
- flowerpower/pipeline/manager.py +21 -642
- flowerpower/pipeline/pipeline.py +571 -0
- flowerpower/pipeline/registry.py +242 -10
- flowerpower/pipeline/visualizer.py +1 -2
- flowerpower/plugins/_io/__init__.py +8 -0
- flowerpower/plugins/mqtt/manager.py +6 -6
- flowerpower/settings/backend.py +0 -2
- flowerpower/settings/job_queue.py +1 -57
- flowerpower/utils/misc.py +0 -256
- flowerpower/utils/monkey.py +1 -83
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
- flowerpower-0.20.0.dist-info/RECORD +58 -0
- flowerpower/fs/__init__.py +0 -29
- flowerpower/fs/base.py +0 -662
- flowerpower/fs/ext.py +0 -2143
- flowerpower/fs/storage_options.py +0 -1420
- flowerpower/job_queue/apscheduler/__init__.py +0 -11
- flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
- flowerpower/job_queue/apscheduler/manager.py +0 -1051
- flowerpower/job_queue/apscheduler/setup.py +0 -554
- flowerpower/job_queue/apscheduler/trigger.py +0 -169
- flowerpower/job_queue/apscheduler/utils.py +0 -311
- flowerpower/pipeline/job_queue.py +0 -583
- flowerpower/pipeline/runner.py +0 -603
- flowerpower/plugins/io/base.py +0 -2520
- flowerpower/plugins/io/helpers/datetime.py +0 -298
- flowerpower/plugins/io/helpers/polars.py +0 -875
- flowerpower/plugins/io/helpers/pyarrow.py +0 -570
- flowerpower/plugins/io/helpers/sql.py +0 -202
- flowerpower/plugins/io/loader/__init__.py +0 -28
- flowerpower/plugins/io/loader/csv.py +0 -37
- flowerpower/plugins/io/loader/deltatable.py +0 -190
- flowerpower/plugins/io/loader/duckdb.py +0 -19
- flowerpower/plugins/io/loader/json.py +0 -37
- flowerpower/plugins/io/loader/mqtt.py +0 -159
- flowerpower/plugins/io/loader/mssql.py +0 -26
- flowerpower/plugins/io/loader/mysql.py +0 -26
- flowerpower/plugins/io/loader/oracle.py +0 -26
- flowerpower/plugins/io/loader/parquet.py +0 -35
- flowerpower/plugins/io/loader/postgres.py +0 -26
- flowerpower/plugins/io/loader/pydala.py +0 -19
- flowerpower/plugins/io/loader/sqlite.py +0 -23
- flowerpower/plugins/io/metadata.py +0 -244
- flowerpower/plugins/io/saver/__init__.py +0 -28
- flowerpower/plugins/io/saver/csv.py +0 -36
- flowerpower/plugins/io/saver/deltatable.py +0 -186
- flowerpower/plugins/io/saver/duckdb.py +0 -19
- flowerpower/plugins/io/saver/json.py +0 -36
- flowerpower/plugins/io/saver/mqtt.py +0 -28
- flowerpower/plugins/io/saver/mssql.py +0 -26
- flowerpower/plugins/io/saver/mysql.py +0 -26
- flowerpower/plugins/io/saver/oracle.py +0 -26
- flowerpower/plugins/io/saver/parquet.py +0 -36
- flowerpower/plugins/io/saver/postgres.py +0 -26
- flowerpower/plugins/io/saver/pydala.py +0 -20
- flowerpower/plugins/io/saver/sqlite.py +0 -24
- flowerpower/utils/scheduler.py +0 -311
- flowerpower-0.11.6.19.dist-info/RECORD +0 -102
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
@@ -1,298 +0,0 @@
|
|
1
|
-
import datetime as dt
|
2
|
-
import re
|
3
|
-
from functools import lru_cache
|
4
|
-
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
5
|
-
|
6
|
-
# import pendulum as pdl
|
7
|
-
import polars as pl
|
8
|
-
import polars.selectors as cs
|
9
|
-
import pyarrow as pa
|
10
|
-
|
11
|
-
|
12
|
-
def get_timestamp_column(df: pl.DataFrame | pl.LazyFrame | pa.Table) -> str | list[str]:
|
13
|
-
if isinstance(df, pa.Table):
|
14
|
-
df = pl.from_arrow(df).lazy()
|
15
|
-
|
16
|
-
# if isinstance(df, pl.LazyFrame):
|
17
|
-
# return df.collect_schema().names()
|
18
|
-
|
19
|
-
return df.select(cs.datetime() | cs.date()).collect_schema().names()
|
20
|
-
|
21
|
-
|
22
|
-
def get_timedelta_str(timedelta_string: str, to: str = "polars") -> str:
|
23
|
-
polars_timedelta_units = [
|
24
|
-
"ns",
|
25
|
-
"us",
|
26
|
-
"ms",
|
27
|
-
"s",
|
28
|
-
"m",
|
29
|
-
"h",
|
30
|
-
"d",
|
31
|
-
"w",
|
32
|
-
"mo",
|
33
|
-
"y",
|
34
|
-
]
|
35
|
-
duckdb_timedelta_units = [
|
36
|
-
"nanosecond",
|
37
|
-
"microsecond",
|
38
|
-
"millisecond",
|
39
|
-
"second",
|
40
|
-
"minute",
|
41
|
-
"hour",
|
42
|
-
"day",
|
43
|
-
"week",
|
44
|
-
"month",
|
45
|
-
"year",
|
46
|
-
]
|
47
|
-
|
48
|
-
unit = re.sub("[0-9]", "", timedelta_string).strip()
|
49
|
-
val = timedelta_string.replace(unit, "").strip()
|
50
|
-
if to == "polars":
|
51
|
-
return (
|
52
|
-
timedelta_string
|
53
|
-
if unit in polars_timedelta_units
|
54
|
-
else val
|
55
|
-
+ dict(zip(duckdb_timedelta_units, polars_timedelta_units))[
|
56
|
-
re.sub("s$", "", unit)
|
57
|
-
]
|
58
|
-
)
|
59
|
-
|
60
|
-
if unit in polars_timedelta_units:
|
61
|
-
return (
|
62
|
-
f"{val} " + dict(zip(polars_timedelta_units, duckdb_timedelta_units))[unit]
|
63
|
-
)
|
64
|
-
|
65
|
-
return f"{val} " + re.sub("s$", "", unit)
|
66
|
-
|
67
|
-
|
68
|
-
# @lru_cache(maxsize=128)
|
69
|
-
# def timestamp_from_string(
|
70
|
-
# timestamp: str,
|
71
|
-
# tz: str | None = None,
|
72
|
-
# exact: bool = True,
|
73
|
-
# strict: bool = False,
|
74
|
-
# naive: bool = False,
|
75
|
-
# ) -> pdl.DateTime | pdl.Date | pdl.Time | dt.datetime | dt.date | dt.time:
|
76
|
-
# """
|
77
|
-
# Converts a string like "2023-01-01 10:00:00" into a datetime.datetime object.
|
78
|
-
|
79
|
-
# Args:
|
80
|
-
# string (str): The string representation of the timestamp, e.g. "2023-01-01 10:00:00".
|
81
|
-
# tz (str, optional): The timezone to use for the timestamp. Defaults to None.
|
82
|
-
# exact (bool, optional): Whether to use exact parsing. Defaults to True.
|
83
|
-
# strict (bool, optional): Whether to use strict parsing. Defaults to False.
|
84
|
-
# naive (bool, optional): Whether to return a naive datetime without a timezone. Defaults to False.
|
85
|
-
|
86
|
-
# Returns:
|
87
|
-
# datetime.datetime: The datetime object.
|
88
|
-
# """
|
89
|
-
# # Extract the timezone from the string if not provided
|
90
|
-
# # tz = extract_timezone(timestamp) if tz is None else tz
|
91
|
-
# # timestamp = timestamp.replace(tz, "").strip() if tz else timestamp
|
92
|
-
|
93
|
-
# pdl_timestamp = pdl.parse(timestamp, exact=exact, strict=strict)
|
94
|
-
|
95
|
-
# if isinstance(pdl_timestamp, pdl.DateTime):
|
96
|
-
# if tz is not None:
|
97
|
-
# pdl_timestamp = pdl_timestamp.naive().set(tz=tz)
|
98
|
-
# if naive or tz is None:
|
99
|
-
# pdl_timestamp = pdl_timestamp.naive()
|
100
|
-
|
101
|
-
# return pdl_timestamp
|
102
|
-
|
103
|
-
|
104
|
-
@lru_cache(maxsize=128)
|
105
|
-
def timestamp_from_string(
|
106
|
-
timestamp_str: str,
|
107
|
-
tz: str | None = None,
|
108
|
-
naive: bool = False,
|
109
|
-
) -> dt.datetime | dt.date | dt.time:
|
110
|
-
"""
|
111
|
-
Converts a timestamp string (ISO 8601 format) into a datetime, date, or time object
|
112
|
-
using only standard Python libraries.
|
113
|
-
|
114
|
-
Handles strings with or without timezone information (e.g., '2023-01-01T10:00:00+02:00',
|
115
|
-
'2023-01-01', '10:00:00'). Supports timezone offsets like '+HH:MM' or '+HHMM'.
|
116
|
-
For named timezones (e.g., 'Europe/Paris'), requires Python 3.9+ and the 'tzdata'
|
117
|
-
package to be installed.
|
118
|
-
|
119
|
-
Args:
|
120
|
-
timestamp_str (str): The string representation of the timestamp (ISO 8601 format).
|
121
|
-
tz (str, optional): Target timezone identifier (e.g., 'UTC', '+02:00', 'Europe/Paris').
|
122
|
-
If provided, the output datetime/time will be localized or converted to this timezone.
|
123
|
-
Defaults to None.
|
124
|
-
naive (bool, optional): If True, return a naive datetime/time (no timezone info),
|
125
|
-
even if the input string or `tz` parameter specifies one. Defaults to False.
|
126
|
-
|
127
|
-
Returns:
|
128
|
-
Union[dt.datetime, dt.date, dt.time]: The parsed datetime, date, or time object.
|
129
|
-
|
130
|
-
Raises:
|
131
|
-
ValueError: If the timestamp string format is invalid or the timezone is
|
132
|
-
invalid/unsupported.
|
133
|
-
"""
|
134
|
-
|
135
|
-
# Regex to parse timezone offsets like +HH:MM or +HHMM
|
136
|
-
_TZ_OFFSET_REGEX = re.compile(r"([+-])(\d{2}):?(\d{2})")
|
137
|
-
|
138
|
-
def _parse_tz_offset(tz_str: str) -> dt.tzinfo | None:
|
139
|
-
"""Parses a timezone offset string into a timezone object."""
|
140
|
-
match = _TZ_OFFSET_REGEX.fullmatch(tz_str)
|
141
|
-
if match:
|
142
|
-
sign, hours, minutes = match.groups()
|
143
|
-
offset_seconds = (int(hours) * 3600 + int(minutes) * 60) * (
|
144
|
-
-1 if sign == "-" else 1
|
145
|
-
)
|
146
|
-
if abs(offset_seconds) >= 24 * 3600:
|
147
|
-
raise ValueError(f"Invalid timezone offset: {tz_str}")
|
148
|
-
return dt.timezone(dt.timedelta(seconds=offset_seconds), name=tz_str)
|
149
|
-
return None
|
150
|
-
|
151
|
-
def _get_tzinfo(tz_identifier: str | None) -> dt.tzinfo | None:
|
152
|
-
"""Gets a tzinfo object from a string (offset or IANA name)."""
|
153
|
-
if tz_identifier is None:
|
154
|
-
return None
|
155
|
-
if tz_identifier.upper() == "UTC":
|
156
|
-
return dt.timezone.utc
|
157
|
-
|
158
|
-
# Try parsing as offset first
|
159
|
-
offset_tz = _parse_tz_offset(tz_identifier)
|
160
|
-
if offset_tz:
|
161
|
-
return offset_tz
|
162
|
-
|
163
|
-
# Try parsing as IANA name using zoneinfo (if available)
|
164
|
-
if ZoneInfo:
|
165
|
-
try:
|
166
|
-
return ZoneInfo(tz_identifier)
|
167
|
-
except ZoneInfoNotFoundError:
|
168
|
-
raise ValueError(
|
169
|
-
f"Timezone '{tz_identifier}' not found. Install 'tzdata' or use offset format."
|
170
|
-
)
|
171
|
-
except Exception as e: # Catch other potential zoneinfo errors
|
172
|
-
raise ValueError(f"Error loading timezone '{tz_identifier}': {e}")
|
173
|
-
else:
|
174
|
-
# zoneinfo not available
|
175
|
-
raise ValueError(
|
176
|
-
f"Invalid timezone: '{tz_identifier}'. Use offset format (e.g., '+02:00') "
|
177
|
-
"or run Python 3.9+ with 'tzdata' installed for named timezones."
|
178
|
-
)
|
179
|
-
|
180
|
-
target_tz: dt.tzinfo | None = _get_tzinfo(tz)
|
181
|
-
parsed_obj: dt.datetime | dt.date | dt.time | None = None
|
182
|
-
|
183
|
-
# Preprocess: Replace space separator, strip whitespace
|
184
|
-
processed_str = timestamp_str.strip().replace(" ", "T")
|
185
|
-
|
186
|
-
# Attempt parsing (datetime, date, time) using fromisoformat
|
187
|
-
try:
|
188
|
-
# Python < 3.11 fromisoformat has limitations (e.g., no Z, no +HHMM offset)
|
189
|
-
# This implementation assumes Python 3.11+ for full ISO 8601 support via fromisoformat
|
190
|
-
# or that input strings use formats compatible with older versions (e.g., +HH:MM)
|
191
|
-
parsed_obj = dt.datetime.fromisoformat(processed_str)
|
192
|
-
except ValueError:
|
193
|
-
try:
|
194
|
-
parsed_obj = dt.date.fromisoformat(processed_str)
|
195
|
-
except ValueError:
|
196
|
-
try:
|
197
|
-
# Time parsing needs care, especially with offsets in older Python
|
198
|
-
parsed_obj = dt.time.fromisoformat(processed_str)
|
199
|
-
except ValueError:
|
200
|
-
# Add fallback for simple HH:MM:SS if needed, though less robust
|
201
|
-
# try:
|
202
|
-
# parsed_obj = dt.datetime.strptime(processed_str, "%H:%M:%S").time()
|
203
|
-
# except ValueError:
|
204
|
-
raise ValueError(f"Invalid timestamp format: '{timestamp_str}'")
|
205
|
-
|
206
|
-
# Apply timezone logic if we have a datetime or time object
|
207
|
-
if isinstance(parsed_obj, (dt.datetime, dt.time)):
|
208
|
-
is_aware = (
|
209
|
-
parsed_obj.tzinfo is not None
|
210
|
-
and parsed_obj.tzinfo.utcoffset(
|
211
|
-
parsed_obj if isinstance(parsed_obj, dt.datetime) else None
|
212
|
-
)
|
213
|
-
is not None
|
214
|
-
)
|
215
|
-
|
216
|
-
if target_tz:
|
217
|
-
if is_aware:
|
218
|
-
# Convert existing aware object to target timezone (only for datetime)
|
219
|
-
if isinstance(parsed_obj, dt.datetime):
|
220
|
-
parsed_obj = parsed_obj.astimezone(target_tz)
|
221
|
-
# else: dt.time cannot be converted without a date context. Keep original tz.
|
222
|
-
else:
|
223
|
-
# Localize naive object to target timezone
|
224
|
-
parsed_obj = parsed_obj.replace(tzinfo=target_tz)
|
225
|
-
is_aware = True # Object is now considered aware
|
226
|
-
|
227
|
-
# Handle naive flag: remove tzinfo if requested
|
228
|
-
if naive and is_aware:
|
229
|
-
parsed_obj = parsed_obj.replace(tzinfo=None)
|
230
|
-
|
231
|
-
# If it's a date object, tz/naive flags are ignored
|
232
|
-
elif isinstance(parsed_obj, dt.date):
|
233
|
-
pass
|
234
|
-
|
235
|
-
return parsed_obj
|
236
|
-
|
237
|
-
|
238
|
-
# def timedelta_from_string(
|
239
|
-
# timedelta_string: str, as_timedelta
|
240
|
-
# ) -> pdl.Duration | dt.timedelta:
|
241
|
-
# """
|
242
|
-
# Converts a string like "2d10s" into a datetime.timedelta object.
|
243
|
-
|
244
|
-
# Args:
|
245
|
-
# string (str): The string representation of the timedelta, e.g. "2d10s".
|
246
|
-
|
247
|
-
# Returns:
|
248
|
-
# datetime.timedelta: The timedelta object.
|
249
|
-
# """
|
250
|
-
# # Extract the numeric value and the unit from the string
|
251
|
-
# matches = re.findall(r"(\d+)([a-zA-Z]+)", timedelta_string)
|
252
|
-
# if not matches:
|
253
|
-
# raise ValueError("Invalid timedelta string")
|
254
|
-
|
255
|
-
# # Initialize the timedelta object
|
256
|
-
# delta = pdl.duration()
|
257
|
-
|
258
|
-
# # Iterate over each match and accumulate the timedelta values
|
259
|
-
# for value, unit in matches:
|
260
|
-
# # Map the unit to the corresponding timedelta attribute
|
261
|
-
# unit_mapping = {
|
262
|
-
# "us": "microseconds",
|
263
|
-
# "ms": "milliseconds",
|
264
|
-
# "s": "seconds",
|
265
|
-
# "m": "minutes",
|
266
|
-
# "h": "hours",
|
267
|
-
# "d": "days",
|
268
|
-
# "w": "weeks",
|
269
|
-
# "mo": "months",
|
270
|
-
# "y": "years",
|
271
|
-
# }
|
272
|
-
# if unit not in unit_mapping:
|
273
|
-
# raise ValueError("Invalid timedelta unit")
|
274
|
-
|
275
|
-
# # Update the timedelta object
|
276
|
-
# kwargs = {unit_mapping[unit]: int(value)}
|
277
|
-
# delta += pdl.duration(**kwargs)
|
278
|
-
|
279
|
-
# return delta.as_timedelta if as_timedelta else delta
|
280
|
-
|
281
|
-
|
282
|
-
# def extract_timezone(timestamp_string):
|
283
|
-
# """
|
284
|
-
# Extracts the timezone from a timestamp string.
|
285
|
-
|
286
|
-
# Args:
|
287
|
-
# timestamp_string (str): The input timestamp string.
|
288
|
-
|
289
|
-
# Returns:
|
290
|
-
# str: The extracted timezone.
|
291
|
-
# """
|
292
|
-
# pattern = r"\b([a-zA-Z]+/{0,1}[a-zA-Z_ ]*)\b" # Matches the timezone portion
|
293
|
-
# match = re.search(pattern, timestamp_string)
|
294
|
-
# if match:
|
295
|
-
# timezone = match.group(0)
|
296
|
-
# return timezone
|
297
|
-
# else:
|
298
|
-
# return None
|