FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. flowerpower/__init__.py +2 -6
  2. flowerpower/cfg/__init__.py +7 -14
  3. flowerpower/cfg/base.py +29 -25
  4. flowerpower/cfg/pipeline/__init__.py +8 -6
  5. flowerpower/cfg/pipeline/_schedule.py +32 -0
  6. flowerpower/cfg/pipeline/adapter.py +0 -5
  7. flowerpower/cfg/pipeline/builder.py +377 -0
  8. flowerpower/cfg/pipeline/run.py +36 -0
  9. flowerpower/cfg/project/__init__.py +11 -24
  10. flowerpower/cfg/project/adapter.py +0 -12
  11. flowerpower/cli/__init__.py +2 -21
  12. flowerpower/cli/cfg.py +0 -3
  13. flowerpower/cli/mqtt.py +0 -6
  14. flowerpower/cli/pipeline.py +22 -415
  15. flowerpower/cli/utils.py +0 -1
  16. flowerpower/flowerpower.py +345 -146
  17. flowerpower/pipeline/__init__.py +2 -0
  18. flowerpower/pipeline/base.py +21 -12
  19. flowerpower/pipeline/io.py +58 -54
  20. flowerpower/pipeline/manager.py +165 -726
  21. flowerpower/pipeline/pipeline.py +643 -0
  22. flowerpower/pipeline/registry.py +285 -18
  23. flowerpower/pipeline/visualizer.py +5 -6
  24. flowerpower/plugins/io/__init__.py +8 -0
  25. flowerpower/plugins/mqtt/__init__.py +7 -11
  26. flowerpower/settings/__init__.py +0 -2
  27. flowerpower/settings/{backend.py → _backend.py} +0 -21
  28. flowerpower/settings/logging.py +1 -1
  29. flowerpower/utils/logging.py +24 -12
  30. flowerpower/utils/misc.py +17 -256
  31. flowerpower/utils/monkey.py +1 -83
  32. flowerpower-0.21.0.dist-info/METADATA +463 -0
  33. flowerpower-0.21.0.dist-info/RECORD +44 -0
  34. flowerpower/cfg/pipeline/schedule.py +0 -74
  35. flowerpower/cfg/project/job_queue.py +0 -238
  36. flowerpower/cli/job_queue.py +0 -1061
  37. flowerpower/fs/__init__.py +0 -29
  38. flowerpower/fs/base.py +0 -662
  39. flowerpower/fs/ext.py +0 -2143
  40. flowerpower/fs/storage_options.py +0 -1420
  41. flowerpower/job_queue/__init__.py +0 -294
  42. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  43. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  44. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  45. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  46. flowerpower/job_queue/apscheduler/setup.py +0 -554
  47. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  48. flowerpower/job_queue/apscheduler/utils.py +0 -311
  49. flowerpower/job_queue/base.py +0 -413
  50. flowerpower/job_queue/rq/__init__.py +0 -10
  51. flowerpower/job_queue/rq/_trigger.py +0 -37
  52. flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
  53. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -231
  54. flowerpower/job_queue/rq/manager.py +0 -1582
  55. flowerpower/job_queue/rq/setup.py +0 -154
  56. flowerpower/job_queue/rq/utils.py +0 -69
  57. flowerpower/mqtt.py +0 -12
  58. flowerpower/pipeline/job_queue.py +0 -583
  59. flowerpower/pipeline/runner.py +0 -603
  60. flowerpower/plugins/io/base.py +0 -2520
  61. flowerpower/plugins/io/helpers/datetime.py +0 -298
  62. flowerpower/plugins/io/helpers/polars.py +0 -875
  63. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  64. flowerpower/plugins/io/helpers/sql.py +0 -202
  65. flowerpower/plugins/io/loader/__init__.py +0 -28
  66. flowerpower/plugins/io/loader/csv.py +0 -37
  67. flowerpower/plugins/io/loader/deltatable.py +0 -190
  68. flowerpower/plugins/io/loader/duckdb.py +0 -19
  69. flowerpower/plugins/io/loader/json.py +0 -37
  70. flowerpower/plugins/io/loader/mqtt.py +0 -159
  71. flowerpower/plugins/io/loader/mssql.py +0 -26
  72. flowerpower/plugins/io/loader/mysql.py +0 -26
  73. flowerpower/plugins/io/loader/oracle.py +0 -26
  74. flowerpower/plugins/io/loader/parquet.py +0 -35
  75. flowerpower/plugins/io/loader/postgres.py +0 -26
  76. flowerpower/plugins/io/loader/pydala.py +0 -19
  77. flowerpower/plugins/io/loader/sqlite.py +0 -23
  78. flowerpower/plugins/io/metadata.py +0 -244
  79. flowerpower/plugins/io/saver/__init__.py +0 -28
  80. flowerpower/plugins/io/saver/csv.py +0 -36
  81. flowerpower/plugins/io/saver/deltatable.py +0 -186
  82. flowerpower/plugins/io/saver/duckdb.py +0 -19
  83. flowerpower/plugins/io/saver/json.py +0 -36
  84. flowerpower/plugins/io/saver/mqtt.py +0 -28
  85. flowerpower/plugins/io/saver/mssql.py +0 -26
  86. flowerpower/plugins/io/saver/mysql.py +0 -26
  87. flowerpower/plugins/io/saver/oracle.py +0 -26
  88. flowerpower/plugins/io/saver/parquet.py +0 -36
  89. flowerpower/plugins/io/saver/postgres.py +0 -26
  90. flowerpower/plugins/io/saver/pydala.py +0 -20
  91. flowerpower/plugins/io/saver/sqlite.py +0 -24
  92. flowerpower/plugins/mqtt/cfg.py +0 -17
  93. flowerpower/plugins/mqtt/manager.py +0 -962
  94. flowerpower/settings/job_queue.py +0 -87
  95. flowerpower/utils/scheduler.py +0 -311
  96. flowerpower-0.11.6.20.dist-info/METADATA +0 -537
  97. flowerpower-0.11.6.20.dist-info/RECORD +0 -102
  98. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/WHEEL +0 -0
  99. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/entry_points.txt +0 -0
  100. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/licenses/LICENSE +0 -0
  101. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/top_level.txt +0 -0
@@ -1,87 +0,0 @@
1
- import os
2
-
3
- from .backend import BACKEND_PROPERTIES
4
- from .executor import EXECUTOR, EXECUTOR_MAX_WORKERS, EXECUTOR_NUM_CPUS
5
-
6
- # WORKER
7
- JOB_QUEUE_TYPE = os.getenv("FP_JOB_QUEUE_TYPE", "rq")
8
-
9
- # RQ WORKER
10
- RQ_BACKEND = os.getenv("FP_RQ_BACKEND", "redis")
11
- RQ_BACKEND_HOST = os.getenv(
12
- "FP_RQ_BACKEND_HOST", BACKEND_PROPERTIES[RQ_BACKEND]["default_host"]
13
- )
14
- RQ_BACKEND_PORT = int(
15
- os.getenv("FP_RQ_BACKEND_PORT", BACKEND_PROPERTIES[RQ_BACKEND]["default_port"])
16
- )
17
- RQ_BACKEND_DB = int(
18
- os.getenv("FP_RQ_BACKEND_DB", BACKEND_PROPERTIES[RQ_BACKEND]["default_database"])
19
- )
20
- RQ_BACKEND_PASSWORD = os.getenv(
21
- "FP_RQ_BACKEND_PASSWORD", BACKEND_PROPERTIES[RQ_BACKEND]["default_password"]
22
- )
23
- RQ_BACKEND_USERNAME = os.getenv(
24
- "FP_RQ_BACKEND_USERNAME", BACKEND_PROPERTIES[RQ_BACKEND]["default_username"]
25
- )
26
- RQ_QUEUES = (
27
- os.getenv("FP_RQ_QUEUES", "default, high, low, scheduler")
28
- .replace(" ", "")
29
- .split(",")
30
- )
31
- RQ_NUM_WORKERS = int(os.getenv("FP_RQ_NUM_WORKERS", EXECUTOR_NUM_CPUS))
32
-
33
- # APS WORKER
34
- APS_BACKEND_DS = os.getenv("FP_APS_BACKEND_DS", "memory")
35
-
36
- APS_BACKEND_DS_HOST = os.getenv(
37
- "FP_APS_BACKEND_DS_HOST",
38
- BACKEND_PROPERTIES.get(APS_BACKEND_DS, {}).get("default_host", None),
39
- )
40
- APS_BACKEND_DS_PORT = int(
41
- os.getenv(
42
- "FP_APS_BACKEND_DS_PORT",
43
- BACKEND_PROPERTIES.get(APS_BACKEND_DS, {}).get("default_port", 0),
44
- )
45
- )
46
- APS_BACKEND_DS_DB = os.getenv(
47
- "FP_APS_BACKEND_DS_DB",
48
- BACKEND_PROPERTIES.get(APS_BACKEND_DS, {}).get("default_database", None),
49
- )
50
- APS_BACKEND_DS_USERNAME = os.getenv(
51
- "FP_APS_BACKEND_DS_USERNAME",
52
- BACKEND_PROPERTIES.get(APS_BACKEND_DS, {}).get("default_username", None),
53
- )
54
- APS_BACKEND_DS_PASSWORD = os.getenv(
55
- "FP_APS_BACKEND_DS_PASSWORD",
56
- BACKEND_PROPERTIES.get(APS_BACKEND_DS, {}).get("default_password", None),
57
- )
58
- APS_BACKEND_DS_SCHEMA = os.getenv("FP_APS_BACKEND_DS_SCHEMA", "flowerpower")
59
-
60
- APS_BACKEND_EB = os.getenv("FP_APS_BACKEND_EB", "memory")
61
- APS_BACKEND_EB_HOST = os.getenv(
62
- "FP_APS_BACKEND_EB_HOST",
63
- BACKEND_PROPERTIES.get(APS_BACKEND_EB, {}).get("default_host", None),
64
- )
65
- APS_BACKEND_EB_PORT = int(
66
- os.getenv(
67
- "FP_APS_BACKEND_EB_PORT",
68
- BACKEND_PROPERTIES.get(APS_BACKEND_EB, {}).get("default_port", 0),
69
- )
70
- )
71
- APS_BACKEND_EB_DB = os.getenv(
72
- "FP_APS_BACKEND_EB_DB",
73
- BACKEND_PROPERTIES.get(APS_BACKEND_EB, {}).get("default_database", None),
74
- )
75
- APS_BACKEND_EB_USERNAME = os.getenv(
76
- "FP_APS_BACKEND_EB_USERNAME",
77
- BACKEND_PROPERTIES.get(APS_BACKEND_EB, {}).get("default_username", None),
78
- )
79
- APS_BACKEND_EB_PASSWORD = os.getenv(
80
- "FP_APS_BACKEND_EB_PASSWORD",
81
- BACKEND_PROPERTIES.get(APS_BACKEND_EB, {}).get("default_password", None),
82
- )
83
-
84
- APS_CLEANUP_INTERVAL = int(os.getenv("FP_APS_CLEANUP_INTERVAL", 300))
85
- APS_MAX_CONCURRENT_JOBS = int(os.getenv("FP_APS_MAX_CONCURRENT_JOBS", 10))
86
- APS_DEFAULT_EXECUTOR = os.getenv("FP_APS_DEFAULT_EXECUTOR", EXECUTOR)
87
- APS_NUM_WORKERS = int(os.getenv("FP_APS_NUM_WORKERS", EXECUTOR_MAX_WORKERS))
@@ -1,311 +0,0 @@
1
- from operator import attrgetter
2
- from typing import List
3
-
4
- from rich.console import Console
5
- from rich.table import Table
6
-
7
-
8
- def humanize_crontab(minute, hour, day, month, day_of_week):
9
- days = {
10
- "0": "Sunday",
11
- "sun": "Sunday",
12
- "7": "Sunday",
13
- "1": "Monday",
14
- "mon": "Monday",
15
- "2": "Tuesday",
16
- "tue": "Tuesday",
17
- "3": "Wednesday",
18
- "wed": "Wednesday",
19
- "4": "Thursday",
20
- "thu": "Thursday",
21
- "5": "Friday",
22
- "fri": "Friday",
23
- "6": "Saturday",
24
- "sat": "Saturday",
25
- "*": "*",
26
- }
27
- months = {
28
- "1": "January",
29
- "2": "February",
30
- "3": "March",
31
- "4": "April",
32
- "5": "May",
33
- "6": "June",
34
- "7": "July",
35
- "8": "August",
36
- "9": "September",
37
- "10": "October",
38
- "11": "November",
39
- "12": "December",
40
- "*": "*",
41
- }
42
-
43
- def get_day_name(day_input):
44
- day_input = str(day_input).lower().strip()
45
- if "-" in day_input:
46
- start, end = day_input.split("-")
47
- return f"{days.get(start.strip(), start)}-{days.get(end.strip(), end)}"
48
- if "," in day_input:
49
- return ", ".join(
50
- days.get(d.strip(), d.strip()) for d in day_input.split(",")
51
- )
52
- return days.get(day_input, day_input)
53
-
54
- try:
55
- minute, hour, day, month, day_of_week = map(
56
- str.strip, map(str, [minute, hour, day, month, day_of_week])
57
- )
58
-
59
- if "/" in minute:
60
- return f"every {minute.split('/')[1]} minutes"
61
- if "/" in hour:
62
- return f"every {hour.split('/')[1]} hours"
63
-
64
- if all(x == "*" for x in [minute, hour, day, month, day_of_week]):
65
- return "every minute"
66
- if [minute, hour, day, month, day_of_week] == ["0", "*", "*", "*", "*"]:
67
- return "every hour"
68
-
69
- if (
70
- minute == "0"
71
- and hour != "*"
72
- and day == "*"
73
- and month == "*"
74
- and day_of_week == "*"
75
- ):
76
- return (
77
- "every day at midnight"
78
- if hour == "0"
79
- else "every day at noon"
80
- if hour == "12"
81
- else f"every day at {hour}:00"
82
- )
83
-
84
- if (
85
- minute == "0"
86
- and hour == "0"
87
- and day == "*"
88
- and month == "*"
89
- and day_of_week != "*"
90
- ):
91
- return f"every {get_day_name(day_of_week)} at midnight"
92
-
93
- if (
94
- minute == "0"
95
- and hour != "*"
96
- and day == "*"
97
- and month == "*"
98
- and day_of_week != "*"
99
- ):
100
- return (
101
- "every weekday at {hour}:00"
102
- if "-" in day_of_week
103
- and "mon" in day_of_week.lower()
104
- and "fri" in day_of_week.lower()
105
- else f"every {get_day_name(day_of_week)} at {hour}:00"
106
- )
107
-
108
- if (
109
- minute != "*"
110
- and hour != "*"
111
- and day == "*"
112
- and month == "*"
113
- and day_of_week == "*"
114
- ):
115
- return f"every day at {hour}:{minute.zfill(2)}"
116
-
117
- if day != "*" and month != "*" and minute == "0" and hour == "0":
118
- return f"on day {day} of {months.get(month, month)} at midnight"
119
-
120
- if (
121
- minute != "*"
122
- and hour == "*"
123
- and day == "*"
124
- and month == "*"
125
- and day_of_week == "*"
126
- ):
127
- return f"every hour at minute {minute}"
128
-
129
- parts = []
130
- if minute != "*":
131
- parts.append(f"at minute {minute}")
132
- if hour != "*":
133
- parts.append(f"hour {hour}")
134
- if day != "*":
135
- parts.append(f"day {day}")
136
- if month != "*":
137
- parts.append(f"month {months.get(month, month)}")
138
- if day_of_week != "*":
139
- parts.append(f"on {get_day_name(day_of_week)}")
140
-
141
- return f"runs {' '.join(parts)}" if parts else "every minute"
142
- except Exception:
143
- return f"{minute} {hour} {day} {month} {day_of_week}"
144
-
145
-
146
- def format_trigger(trigger):
147
- trigger_type = trigger.__class__.__name__
148
-
149
- if trigger_type == "IntervalTrigger":
150
- for unit in ["seconds", "minutes", "hours", "days"]:
151
- if value := getattr(trigger, unit, None):
152
- return f"Interval: Every {value}{unit[0]}"
153
- return "Interval"
154
-
155
- if trigger_type == "CronTrigger":
156
- try:
157
- cron_parts = dict(
158
- part.split("=")
159
- for part in str(trigger).strip("CronTrigger(").rstrip(")").split(", ")
160
- )
161
- cron_parts = {k: v.strip("'") for k, v in cron_parts.items()}
162
- crontab = f"{cron_parts['minute']} {cron_parts['hour']} {cron_parts['day']} {cron_parts['month']} {cron_parts['day_of_week']}"
163
- human_readable = humanize_crontab(
164
- **{
165
- k: cron_parts[k]
166
- for k in ["minute", "hour", "day", "month", "day_of_week"]
167
- }
168
- )
169
- return f"Cron: {human_readable} ({crontab})"
170
- except Exception:
171
- return f"Cron: {str(trigger)}"
172
-
173
- if trigger_type == "DateTrigger":
174
- return f"Date: Once at {trigger.run_date.strftime('%Y-%m-%d %H:%M:%S')}"
175
-
176
- return f"{trigger_type}: {str(trigger)}"
177
-
178
-
179
- def display_schedules(schedules: List):
180
- console = Console()
181
- total_width = console.width - 10
182
-
183
- width_ratios = {
184
- "id": 0.20,
185
- "task": 0.10,
186
- "trigger": 0.25,
187
- "name": 0.15,
188
- "run_args": 0.15,
189
- "next_fire": 0.08,
190
- "last_fire": 0.08,
191
- "paused": 0.01,
192
- }
193
-
194
- widths = {k: max(10, int(total_width * ratio)) for k, ratio in width_ratios.items()}
195
-
196
- table = Table(
197
- show_header=True,
198
- header_style="bold magenta",
199
- width=total_width,
200
- row_styles=["", "dim"],
201
- border_style="blue",
202
- show_lines=True,
203
- )
204
-
205
- for col, style, width in [
206
- ("ID", "dim", widths["id"]),
207
- ("Task", "cyan", widths["task"]),
208
- ("Trigger", "blue", widths["trigger"]),
209
- ("Name", "yellow", widths["name"]),
210
- ("Run Args", "yellow", widths["run_args"]),
211
- ("Next Fire Time", "green", widths["next_fire"]),
212
- ("Last Fire Time", "red", widths["last_fire"]),
213
- ("Paused", "bold", widths["paused"]),
214
- ]:
215
- table.add_column(col, style=style, width=width)
216
-
217
- for schedule in sorted(schedules, key=attrgetter("next_fire_time")):
218
- table.add_row(
219
- schedule.id,
220
- schedule.task_id.split(":")[-1],
221
- format_trigger(schedule.trigger),
222
- (
223
- str(schedule.args[1])
224
- if schedule.args and len(schedule.args) > 1
225
- else "None"
226
- ),
227
- "\n".join(f"{k}: {v}" for k, v in (schedule.kwargs or {}).items())
228
- or "None",
229
- (
230
- schedule.next_fire_time.strftime("%Y-%m-%d %H:%M:%S")
231
- if schedule.next_fire_time
232
- else "Never"
233
- ),
234
- (
235
- schedule.last_fire_time.strftime("%Y-%m-%d %H:%M:%S")
236
- if schedule.last_fire_time
237
- else "Never"
238
- ),
239
- "✓" if schedule.paused else "✗",
240
- )
241
-
242
- console.print(table)
243
-
244
-
245
- def display_tasks(tasks):
246
- console = Console()
247
- table = Table(title="Tasks")
248
-
249
- widths = {"id": 50, "executor": 15, "max_jobs": 15, "misfire": 20}
250
-
251
- for col, style, width in [
252
- ("ID", "cyan", widths["id"]),
253
- ("Job Executor", "blue", widths["executor"]),
254
- ("Max Running Jobs", "yellow", widths["max_jobs"]),
255
- ("Misfire Grace Time", "green", widths["misfire"]),
256
- ]:
257
- table.add_column(col, style=style, width=width)
258
-
259
- for task in sorted(tasks, key=attrgetter("id")):
260
- table.add_row(
261
- task.id,
262
- str(task.job_executor),
263
- str(task.max_running_jobs or "None"),
264
- str(task.misfire_grace_time or "None"),
265
- )
266
-
267
- console.print(table)
268
-
269
-
270
- def display_jobs(jobs):
271
- console = Console()
272
- table = Table(title="Jobs")
273
-
274
- widths = {
275
- "id": 10,
276
- "task_id": 40,
277
- "args": 20,
278
- "kwargs": 20,
279
- "schedule": 15,
280
- "created": 25,
281
- "status": 15,
282
- }
283
-
284
- for col, style, width in [
285
- ("ID", "cyan", widths["id"]),
286
- ("Task ID", "blue", widths["task_id"]),
287
- ("Args", "yellow", widths["args"]),
288
- ("Kwargs", "yellow", widths["kwargs"]),
289
- ("Schedule ID", "green", widths["schedule"]),
290
- ("Created At", "magenta", widths["created"]),
291
- ("Status", "red", widths["status"]),
292
- ]:
293
- table.add_column(col, style=style, width=width)
294
-
295
- for job in sorted(jobs, key=attrgetter("id")):
296
- status = "Running" if job.acquired_by else "Pending"
297
- table.add_row(
298
- str(job.id),
299
- job.task_id,
300
- str(job.args if job.args else "None"),
301
- (
302
- "\n".join(f"{k}: {v}" for k, v in job.kwargs.items())
303
- if job.kwargs
304
- else "None"
305
- ),
306
- str(job.schedule_id or "None"),
307
- job.created_at.strftime("%Y-%m-%d %H:%M:%S"),
308
- status,
309
- )
310
-
311
- console.print(table)